From f9db8d46be58b4bb742576cc27f9deaa93fd29aa Mon Sep 17 00:00:00 2001 From: shubham1206agra Date: Tue, 10 May 2022 13:56:01 +0530 Subject: [PATCH] Revert "Squashed commit of the following:" This reverts commit 1ea72a60aba0045995fd651927f736334b910ad4. --- .ci/linux-steps.yaml | 1 - .ci/macos-steps.yaml | 1 - .github/workflows/main.yml | 1 - CMake/FindGo.cmake | 4 +- CMake/FindGonum.cmake | 11 +- COPYRIGHT.txt | 2 +- HISTORY.md | 2 - src/mlpack/bindings/python/copy_artifacts.py | 12 +- .../bindings/python/is_serializable.hpp | 4 +- src/mlpack/core/cereal/CMakeLists.txt | 2 + .../core/cereal/pointer_variant_wrapper.hpp | 159 + .../cereal/pointer_vector_variant_wrapper.hpp | 97 + src/mlpack/core/util/size_checks.hpp | 19 +- src/mlpack/methods/ann/CMakeLists.txt | 11 +- .../methods/ann/{not_adapted => }/brnn.hpp | 1 + .../ann/{not_adapted => }/brnn_impl.hpp | 0 .../convolution_rules/naive_convolution.hpp | 51 +- src/mlpack/methods/ann/ffn.hpp | 688 +- src/mlpack/methods/ann/ffn_impl.hpp | 1278 ++-- src/mlpack/methods/ann/forward_decls.hpp | 29 - .../ann/{not_adapted => }/gan/CMakeLists.txt | 0 .../methods/ann/{not_adapted => }/gan/gan.hpp | 0 .../ann/{not_adapted => }/gan/gan_impl.hpp | 0 .../{not_adapted => }/gan/gan_policies.hpp | 0 .../gan/metrics/CMakeLists.txt | 0 .../gan/metrics/inception_score.hpp | 0 .../gan/metrics/inception_score_impl.hpp | 0 .../ann/{not_adapted => }/gan/wgan_impl.hpp | 0 .../ann/{not_adapted => }/gan/wgangp_impl.hpp | 0 .../methods/ann/init_rules/const_init.hpp | 8 +- .../methods/ann/init_rules/glorot_init.hpp | 18 +- src/mlpack/methods/ann/init_rules/he_init.hpp | 6 - .../methods/ann/init_rules/network_init.hpp | 47 +- .../methods/ann/init_rules/oivs_init.hpp | 10 +- .../methods/ann/init_rules/random_init.hpp | 7 - src/mlpack/methods/ann/layer/CMakeLists.txt | 112 +- .../adaptive_max_pooling.hpp | 90 +- .../adaptive_max_pooling_impl.hpp | 42 +- .../adaptive_mean_pooling.hpp | 91 +- .../adaptive_mean_pooling_impl.hpp | 42 +- src/mlpack/methods/ann/layer/add.hpp | 116 +- src/mlpack/methods/ann/layer/add_impl.hpp | 108 +- .../ann/layer/{not_adapted => }/add_merge.hpp | 138 +- .../methods/ann/layer/add_merge_impl.hpp | 168 + .../methods/ann/layer/alpha_dropout.hpp | 79 +- .../methods/ann/layer/alpha_dropout_impl.hpp | 96 +- .../{not_adapted => }/atrous_convolution.hpp | 117 +- .../atrous_convolution_impl.hpp | 149 +- src/mlpack/methods/ann/layer/base_layer.hpp | 267 +- .../layer/{not_adapted => }/batch_norm.hpp | 92 +- .../{not_adapted => }/batch_norm_impl.hpp | 122 +- .../bicubic_interpolation.hpp | 0 .../bicubic_interpolation_impl.hpp | 0 .../bilinear_interpolation.hpp | 96 +- .../bilinear_interpolation_impl.hpp | 127 +- .../ann/layer/{not_adapted => }/c_relu.hpp | 60 +- .../layer/{not_adapted => }/c_relu_impl.hpp | 25 +- .../ann/layer/{not_adapted => }/celu.hpp | 67 +- .../ann/layer/{not_adapted => }/celu_impl.hpp | 31 +- .../{not_adapted => }/channel_shuffle.hpp | 0 .../channel_shuffle_impl.hpp | 0 src/mlpack/methods/ann/layer/concat.hpp | 263 + src/mlpack/methods/ann/layer/concat_impl.hpp | 293 + .../{not_adapted => }/concat_performance.hpp | 45 +- .../concat_performance_impl.hpp | 69 +- src/mlpack/methods/ann/layer/concatenate.hpp | 102 +- .../methods/ann/layer/concatenate_impl.hpp | 123 +- .../ann/layer/{not_adapted => }/constant.hpp | 82 +- .../methods/ann/layer/constant_impl.hpp | 65 + src/mlpack/methods/ann/layer/convolution.hpp | 289 +- .../methods/ann/layer/convolution_impl.hpp | 945 ++- src/mlpack/methods/ann/layer/dropconnect.hpp | 149 +- .../methods/ann/layer/dropconnect_impl.hpp | 166 +- src/mlpack/methods/ann/layer/dropout.hpp | 83 +- src/mlpack/methods/ann/layer/dropout_impl.hpp | 96 +- .../ann/layer/{not_adapted => }/elu.hpp | 76 +- .../ann/layer/{not_adapted => }/elu_impl.hpp | 57 +- .../ann/layer/{not_adapted => }/fast_lstm.hpp | 133 +- .../{not_adapted => }/fast_lstm_impl.hpp | 86 +- .../{not_adapted => }/flatten_t_swish.hpp | 0 .../flatten_t_swish_impl.hpp | 0 .../layer/{not_adapted => }/flexible_relu.hpp | 115 +- .../{not_adapted => }/flexible_relu_impl.hpp | 61 +- .../ann/layer/{not_adapted => }/glimpse.hpp | 115 +- .../layer/{not_adapted => }/glimpse_impl.hpp | 59 +- .../layer/{not_adapted => }/group_norm.hpp | 0 .../{not_adapted => }/group_norm_impl.hpp | 0 .../ann/layer/{not_adapted => }/gru.hpp | 100 +- src/mlpack/methods/ann/layer/gru_impl.hpp | 411 ++ .../ann/layer/{not_adapted => }/hard_tanh.hpp | 52 +- .../{not_adapted => }/hard_tanh_impl.hpp | 21 +- .../layer/{not_adapted => }/hardshrink.hpp | 60 +- .../{not_adapted => }/hardshrink_impl.hpp | 24 +- src/mlpack/methods/ann/layer/highway.hpp | 270 + src/mlpack/methods/ann/layer/highway_impl.hpp | 238 + .../layer/{not_adapted => }/instance_norm.hpp | 0 .../{not_adapted => }/instance_norm_impl.hpp | 0 .../ann/layer/{not_adapted => }/isrlu.hpp | 0 .../layer/{not_adapted => }/isrlu_impl.hpp | 0 .../ann/layer/{not_adapted => }/join.hpp | 55 +- .../ann/layer/{not_adapted => }/join_impl.hpp | 26 +- src/mlpack/methods/ann/layer/layer.hpp | 377 +- .../layer/{not_adapted => }/layer_norm.hpp | 93 +- .../{not_adapted => }/layer_norm_impl.hpp | 62 +- src/mlpack/methods/ann/layer/layer_traits.hpp | 130 + src/mlpack/methods/ann/layer/layer_types.hpp | 298 +- src/mlpack/methods/ann/layer/leaky_relu.hpp | 71 +- .../methods/ann/layer/leaky_relu_impl.hpp | 71 +- src/mlpack/methods/ann/layer/linear.hpp | 172 +- src/mlpack/methods/ann/layer/linear3d.hpp | 141 +- .../methods/ann/layer/linear3d_impl.hpp | 198 +- src/mlpack/methods/ann/layer/linear_impl.hpp | 135 +- .../methods/ann/layer/linear_no_bias.hpp | 137 +- .../methods/ann/layer/linear_no_bias_impl.hpp | 127 +- src/mlpack/methods/ann/layer/log_softmax.hpp | 71 +- .../methods/ann/layer/log_softmax_impl.hpp | 70 +- .../ann/layer/{not_adapted => }/lookup.hpp | 83 +- .../layer/{not_adapted => }/lookup_impl.hpp | 60 +- .../layer/{not_adapted => }/lp_pooling.hpp | 0 .../{not_adapted => }/lp_pooling_impl.hpp | 0 src/mlpack/methods/ann/layer/lstm.hpp | 347 +- src/mlpack/methods/ann/layer/lstm_impl.hpp | 570 +- src/mlpack/methods/ann/layer/max_pooling.hpp | 286 +- .../methods/ann/layer/max_pooling_impl.hpp | 244 +- .../layer/{not_adapted => }/mean_pooling.hpp | 191 +- .../methods/ann/layer/mean_pooling_impl.hpp | 134 + .../minibatch_discrimination.hpp | 99 +- .../layer/minibatch_discrimination_impl.hpp | 147 + src/mlpack/methods/ann/layer/multi_layer.hpp | 252 - .../methods/ann/layer/multi_layer_impl.hpp | 414 -- .../{not_adapted => }/multihead_attention.hpp | 133 +- .../multihead_attention_impl.hpp | 154 +- .../{not_adapted => }/multiply_constant.hpp | 63 +- .../ann/layer/multiply_constant_impl.hpp | 96 + .../{not_adapted => }/multiply_merge.hpp | 119 +- .../methods/ann/layer/multiply_merge_impl.hpp | 190 + .../nearest_interpolation.hpp | 0 .../nearest_interpolation_impl.hpp | 0 src/mlpack/methods/ann/layer/noisylinear.hpp | 161 +- .../methods/ann/layer/noisylinear_impl.hpp | 173 +- .../methods/ann/layer/not_adapted/README.md | 9 - .../ann/layer/not_adapted/add_merge_impl.hpp | 145 - .../methods/ann/layer/not_adapted/concat.hpp | 222 - .../ann/layer/not_adapted/concat_impl.hpp | 267 - .../ann/layer/not_adapted/constant_impl.hpp | 118 - .../ann/layer/not_adapted/gru_impl.hpp | 369 -- .../methods/ann/layer/not_adapted/highway.hpp | 157 - .../ann/layer/not_adapted/highway_impl.hpp | 194 - .../layer/not_adapted/mean_pooling_impl.hpp | 108 - .../minibatch_discrimination_impl.hpp | 138 - .../not_adapted/multiply_constant_impl.hpp | 56 - .../layer/not_adapted/multiply_merge_impl.hpp | 116 - .../not_adapted/recurrent_attention_impl.hpp | 237 - .../ann/layer/not_adapted/recurrent_impl.hpp | 284 - .../not_adapted/reparametrization_impl.hpp | 152 - .../ann/layer/not_adapted/sequential.hpp | 150 - .../ann/layer/not_adapted/sequential_impl.hpp | 163 - .../layer/not_adapted/weight_norm_impl.hpp | 202 - src/mlpack/methods/ann/layer/padding.hpp | 107 +- src/mlpack/methods/ann/layer/padding_impl.hpp | 140 +- .../{not_adapted => }/parametric_relu.hpp | 75 +- .../parametric_relu_impl.hpp | 62 +- .../layer/{not_adapted => }/pixel_shuffle.hpp | 0 .../{not_adapted => }/pixel_shuffle_impl.hpp | 0 .../{not_adapted => }/positional_encoding.hpp | 61 +- .../positional_encoding_impl.hpp | 38 +- .../ann/layer/radial_basis_function.hpp | 120 +- .../ann/layer/radial_basis_function_impl.hpp | 140 +- .../ann/layer/{not_adapted => }/recurrent.hpp | 112 +- .../{not_adapted => }/recurrent_attention.hpp | 128 +- .../ann/layer/recurrent_attention_impl.hpp | 226 + .../methods/ann/layer/recurrent_impl.hpp | 361 ++ .../methods/ann/layer/recurrent_layer.hpp | 107 - .../ann/layer/recurrent_layer_impl.hpp | 84 - .../{not_adapted => }/reinforce_normal.hpp | 56 +- .../reinforce_normal_impl.hpp | 25 +- .../ann/layer/{not_adapted => }/relu6.hpp | 0 .../layer/{not_adapted => }/relu6_impl.hpp | 0 .../{not_adapted => }/reparametrization.hpp | 143 +- .../ann/layer/reparametrization_impl.hpp | 156 + .../ann/layer/{not_adapted => }/select.hpp | 76 +- .../layer/{not_adapted => }/select_impl.hpp | 35 +- src/mlpack/methods/ann/layer/sequential.hpp | 267 + .../methods/ann/layer/sequential_impl.hpp | 270 + .../methods/ann/layer/serialization.hpp | 53 - .../ann/layer/{not_adapted => }/softmax.hpp | 62 +- .../layer/{not_adapted => }/softmax_impl.hpp | 26 +- .../ann/layer/{not_adapted => }/softmin.hpp | 62 +- .../layer/{not_adapted => }/softmin_impl.hpp | 26 +- .../layer/{not_adapted => }/softshrink.hpp | 62 +- .../{not_adapted => }/softshrink_impl.hpp | 28 +- .../{not_adapted => }/spatial_dropout.hpp | 71 +- .../spatial_dropout_impl.hpp | 64 +- .../ann/layer/{not_adapted => }/subview.hpp | 94 +- .../transposed_convolution.hpp | 211 +- .../transposed_convolution_impl.hpp | 175 +- .../{not_adapted => }/virtual_batch_norm.hpp | 119 +- .../virtual_batch_norm_impl.hpp | 72 +- .../vr_class_reward.hpp | 80 +- .../ann/layer/vr_class_reward_impl.hpp | 107 + .../layer/{not_adapted => }/weight_norm.hpp | 129 +- .../methods/ann/layer/weight_norm_impl.hpp | 165 + .../methods/ann/loss_functions/CMakeLists.txt | 2 - .../binary_cross_entropy_loss.hpp | 56 +- .../binary_cross_entropy_loss_impl.hpp | 31 +- .../loss_functions/cosine_embedding_loss.hpp | 60 +- .../cosine_embedding_loss_impl.hpp | 43 +- .../methods/ann/loss_functions/dice_loss.hpp | 42 +- .../ann/loss_functions/dice_loss_impl.hpp | 41 +- .../loss_functions/earth_mover_distance.hpp | 42 +- .../earth_mover_distance_impl.hpp | 34 +- .../methods/ann/loss_functions/empty_loss.hpp | 36 +- .../ann/loss_functions/empty_loss_impl.hpp | 22 +- .../loss_functions/hinge_embedding_loss.hpp | 43 +- .../hinge_embedding_loss_impl.hpp | 35 +- .../methods/ann/loss_functions/hinge_loss.hpp | 44 +- .../ann/loss_functions/hinge_loss_impl.hpp | 39 +- .../methods/ann/loss_functions/huber_loss.hpp | 42 +- .../ann/loss_functions/huber_loss_impl.hpp | 47 +- .../ann/loss_functions/kl_divergence.hpp | 43 +- .../ann/loss_functions/kl_divergence_impl.hpp | 35 +- .../methods/ann/loss_functions/l1_loss.hpp | 42 +- .../ann/loss_functions/l1_loss_impl.hpp | 37 +- .../ann/loss_functions/log_cosh_loss.hpp | 40 +- .../ann/loss_functions/log_cosh_loss_impl.hpp | 37 +- .../loss_functions/margin_ranking_loss.hpp | 51 +- .../margin_ranking_loss_impl.hpp | 50 +- .../mean_absolute_percentage_error.hpp | 47 +- .../mean_absolute_percentage_error_impl.hpp | 37 +- .../ann/loss_functions/mean_bias_error.hpp | 46 +- .../loss_functions/mean_bias_error_impl.hpp | 35 +- .../ann/loss_functions/mean_squared_error.hpp | 43 +- .../mean_squared_error_impl.hpp | 33 +- .../mean_squared_logarithmic_error.hpp | 46 +- .../mean_squared_logarithmic_error_impl.hpp | 34 +- .../multilabel_softmargin_loss.hpp | 66 +- .../multilabel_softmargin_loss_impl.hpp | 38 +- .../negative_log_likelihood.hpp | 64 +- .../negative_log_likelihood_impl.hpp | 37 +- .../ann/loss_functions/poisson_nll_loss.hpp | 65 +- .../loss_functions/poisson_nll_loss_impl.hpp | 41 +- .../loss_functions/reconstruction_loss.hpp | 44 +- .../reconstruction_loss_impl.hpp | 38 +- .../sigmoid_cross_entropy_error.hpp | 48 +- .../sigmoid_cross_entropy_error_impl.hpp | 37 +- .../ann/loss_functions/soft_margin_loss.hpp | 48 +- .../loss_functions/soft_margin_loss_impl.hpp | 39 +- .../loss_functions/triplet_margin_loss.hpp | 48 +- .../triplet_margin_loss_impl.hpp | 41 +- .../loss_functions/vr_class_reward_impl.hpp | 104 - src/mlpack/methods/ann/make_alias.hpp | 55 - .../ann/{not_adapted => }/rbm/CMakeLists.txt | 0 .../methods/ann/{not_adapted => }/rbm/rbm.hpp | 0 .../ann/{not_adapted => }/rbm/rbm_impl.hpp | 0 .../{not_adapted => }/rbm/rbm_policies.hpp | 0 .../rbm/spike_slab_rbm_impl.hpp | 0 .../methods/ann/regularizer/lregularizer.hpp | 2 +- .../ann/regularizer/no_regularizer.hpp | 6 - .../regularizer/orthogonal_regularizer.hpp | 2 +- src/mlpack/methods/ann/rnn.hpp | 498 +- src/mlpack/methods/ann/rnn_impl.hpp | 968 +-- src/mlpack/methods/ann/util/CMakeLists.txt | 14 + .../methods/ann/util/check_input_shape.hpp | 53 + src/mlpack/methods/ann/visitor/CMakeLists.txt | 71 + .../methods/ann/visitor/add_visitor.hpp | 64 + .../methods/ann/visitor/add_visitor_impl.hpp | 64 + .../methods/ann/visitor/backward_visitor.hpp | 85 + .../ann/visitor/backward_visitor_impl.hpp | 84 + .../methods/ann/visitor/bias_set_visitor.hpp | 82 + .../ann/visitor/bias_set_visitor_impl.hpp | 101 + .../methods/ann/visitor/copy_visitor.hpp | 41 + .../methods/ann/visitor/copy_visitor_impl.hpp | 39 + .../methods/ann/visitor/delete_visitor.hpp | 51 + .../ann/visitor/delete_visitor_impl.hpp | 53 + .../methods/ann/visitor/delta_visitor.hpp | 43 + .../ann/visitor/delta_visitor_impl.hpp | 36 + .../ann/visitor/deterministic_set_visitor.hpp | 83 + .../deterministic_set_visitor_impl.hpp | 88 + .../methods/ann/visitor/forward_visitor.hpp | 54 + .../ann/visitor/forward_visitor_impl.hpp | 43 + .../ann/visitor/gradient_set_visitor.hpp | 82 + .../ann/visitor/gradient_set_visitor_impl.hpp | 100 + .../ann/visitor/gradient_update_visitor.hpp | 82 + .../visitor/gradient_update_visitor_impl.hpp | 106 + .../methods/ann/visitor/gradient_visitor.hpp | 89 + .../ann/visitor/gradient_visitor_impl.hpp | 90 + .../ann/visitor/gradient_zero_visitor.hpp | 60 + .../visitor/gradient_zero_visitor_impl.hpp | 57 + .../ann/visitor/input_shape_visitor.hpp | 58 + .../ann/visitor/input_shape_visitor_impl.hpp | 53 + .../visitor/load_output_parameter_visitor.hpp | 65 + .../load_output_parameter_visitor_impl.hpp | 66 + .../methods/ann/visitor/loss_visitor.hpp | 71 + .../methods/ann/visitor/loss_visitor_impl.hpp | 99 + .../ann/visitor/output_height_visitor.hpp | 75 + .../visitor/output_height_visitor_impl.hpp | 99 + .../ann/visitor/output_parameter_visitor.hpp | 43 + .../visitor/output_parameter_visitor_impl.hpp | 36 + .../ann/visitor/output_width_visitor.hpp | 75 + .../ann/visitor/output_width_visitor_impl.hpp | 99 + .../ann/visitor/parameters_set_visitor.hpp | 64 + .../visitor/parameters_set_visitor_impl.hpp | 58 + .../ann/visitor/parameters_visitor.hpp | 64 + .../ann/visitor/parameters_visitor_impl.hpp | 58 + .../ann/visitor/reset_cell_visitor.hpp | 62 + .../ann/visitor/reset_cell_visitor_impl.hpp | 58 + .../methods/ann/visitor/reset_visitor.hpp | 75 + .../ann/visitor/reset_visitor_impl.hpp | 80 + .../ann/visitor/reward_set_visitor.hpp | 81 + .../ann/visitor/reward_set_visitor_impl.hpp | 87 + .../methods/ann/visitor/run_set_visitor.hpp | 83 + .../ann/visitor/run_set_visitor_impl.hpp | 88 + .../visitor/save_output_parameter_visitor.hpp | 64 + .../save_output_parameter_visitor_impl.hpp | 64 + .../ann/visitor/set_input_height_visitor.hpp | 84 + .../visitor/set_input_height_visitor_impl.hpp | 102 + .../ann/visitor/set_input_width_visitor.hpp | 83 + .../visitor/set_input_width_visitor_impl.hpp | 102 + .../ann/visitor/weight_set_visitor.hpp | 82 + .../ann/visitor/weight_set_visitor_impl.hpp | 100 + .../ann/visitor/weight_size_visitor.hpp | 76 + .../ann/visitor/weight_size_visitor_impl.hpp | 84 + src/mlpack/methods/kmeans/kmeans_impl.hpp | 17 +- .../linear_regression/linear_regression.cpp | 17 +- src/mlpack/methods/perceptron/perceptron.hpp | 23 - .../methods/perceptron/perceptron_impl.hpp | 29 +- .../q_learning_impl.hpp | 4 +- .../q_networks/dueling_dqn.hpp | 2 +- .../q_networks/simple_dqn.hpp | 34 +- .../methods/reinforcement_learning/sac.hpp | 1 + src/mlpack/prereqs.hpp | 12 +- src/mlpack/tests/CMakeLists.txt | 18 +- .../tests/activation_functions_test.cpp | 146 +- src/mlpack/tests/ann_layer_test.cpp | 5652 +++++++---------- src/mlpack/tests/ann_test_tools.hpp | 41 +- src/mlpack/tests/ann_visitor_test.cpp | 235 + src/mlpack/tests/async_learning_test.cpp | 32 +- src/mlpack/tests/callback_test.cpp | 53 +- src/mlpack/tests/catch.hpp | 14 +- src/mlpack/tests/convolution_test.cpp | 200 - .../tests/convolutional_network_test.cpp | 353 +- src/mlpack/tests/custom_layer.hpp | 24 +- src/mlpack/tests/cv_test.cpp | 36 +- .../tests/feedforward_network_2_test.cpp | 13 +- src/mlpack/tests/feedforward_network_test.cpp | 713 ++- .../tests/{not_adapted => }/gan_test.cpp | 0 src/mlpack/tests/init_rules_test.cpp | 58 +- src/mlpack/tests/ksinit_test.cpp | 12 +- src/mlpack/tests/layer_names_test.cpp | 160 + src/mlpack/tests/loss_functions_test.cpp | 169 +- src/mlpack/tests/perceptron_test.cpp | 34 +- .../{not_adapted => }/rbm_network_test.cpp | 0 src/mlpack/tests/recurrent_network_test.cpp | 320 +- src/mlpack/tests/reward_clipping_test.cpp | 4 +- src/mlpack/tests/serialization_test.cpp | 8 +- .../tests/{not_adapted => }/wgan_test.cpp | 0 356 files changed, 22397 insertions(+), 16978 deletions(-) create mode 100644 src/mlpack/core/cereal/pointer_variant_wrapper.hpp create mode 100644 src/mlpack/core/cereal/pointer_vector_variant_wrapper.hpp rename src/mlpack/methods/ann/{not_adapted => }/brnn.hpp (99%) rename src/mlpack/methods/ann/{not_adapted => }/brnn_impl.hpp (100%) delete mode 100644 src/mlpack/methods/ann/forward_decls.hpp rename src/mlpack/methods/ann/{not_adapted => }/gan/CMakeLists.txt (100%) rename src/mlpack/methods/ann/{not_adapted => }/gan/gan.hpp (100%) rename src/mlpack/methods/ann/{not_adapted => }/gan/gan_impl.hpp (100%) rename src/mlpack/methods/ann/{not_adapted => }/gan/gan_policies.hpp (100%) rename src/mlpack/methods/ann/{not_adapted => }/gan/metrics/CMakeLists.txt (100%) rename src/mlpack/methods/ann/{not_adapted => }/gan/metrics/inception_score.hpp (100%) rename src/mlpack/methods/ann/{not_adapted => }/gan/metrics/inception_score_impl.hpp (100%) rename src/mlpack/methods/ann/{not_adapted => }/gan/wgan_impl.hpp (100%) rename src/mlpack/methods/ann/{not_adapted => }/gan/wgangp_impl.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/adaptive_max_pooling.hpp (62%) rename src/mlpack/methods/ann/layer/{not_adapted => }/adaptive_max_pooling_impl.hpp (55%) rename src/mlpack/methods/ann/layer/{not_adapted => }/adaptive_mean_pooling.hpp (62%) rename src/mlpack/methods/ann/layer/{not_adapted => }/adaptive_mean_pooling_impl.hpp (55%) rename src/mlpack/methods/ann/layer/{not_adapted => }/add_merge.hpp (52%) create mode 100644 src/mlpack/methods/ann/layer/add_merge_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/atrous_convolution.hpp (82%) rename src/mlpack/methods/ann/layer/{not_adapted => }/atrous_convolution_impl.hpp (81%) rename src/mlpack/methods/ann/layer/{not_adapted => }/batch_norm.hpp (69%) rename src/mlpack/methods/ann/layer/{not_adapted => }/batch_norm_impl.hpp (65%) rename src/mlpack/methods/ann/layer/{not_adapted => }/bicubic_interpolation.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/bicubic_interpolation_impl.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/bilinear_interpolation.hpp (55%) rename src/mlpack/methods/ann/layer/{not_adapted => }/bilinear_interpolation_impl.hpp (51%) rename src/mlpack/methods/ann/layer/{not_adapted => }/c_relu.hpp (65%) rename src/mlpack/methods/ann/layer/{not_adapted => }/c_relu_impl.hpp (63%) rename src/mlpack/methods/ann/layer/{not_adapted => }/celu.hpp (64%) rename src/mlpack/methods/ann/layer/{not_adapted => }/celu_impl.hpp (66%) rename src/mlpack/methods/ann/layer/{not_adapted => }/channel_shuffle.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/channel_shuffle_impl.hpp (100%) create mode 100644 src/mlpack/methods/ann/layer/concat.hpp create mode 100644 src/mlpack/methods/ann/layer/concat_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/concat_performance.hpp (67%) rename src/mlpack/methods/ann/layer/{not_adapted => }/concat_performance_impl.hpp (61%) rename src/mlpack/methods/ann/layer/{not_adapted => }/constant.hpp (56%) create mode 100644 src/mlpack/methods/ann/layer/constant_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/elu.hpp (70%) rename src/mlpack/methods/ann/layer/{not_adapted => }/elu_impl.hpp (53%) rename src/mlpack/methods/ann/layer/{not_adapted => }/fast_lstm.hpp (72%) rename src/mlpack/methods/ann/layer/{not_adapted => }/fast_lstm_impl.hpp (81%) rename src/mlpack/methods/ann/layer/{not_adapted => }/flatten_t_swish.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/flatten_t_swish_impl.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/flexible_relu.hpp (50%) rename src/mlpack/methods/ann/layer/{not_adapted => }/flexible_relu_impl.hpp (51%) rename src/mlpack/methods/ann/layer/{not_adapted => }/glimpse.hpp (80%) rename src/mlpack/methods/ann/layer/{not_adapted => }/glimpse_impl.hpp (76%) rename src/mlpack/methods/ann/layer/{not_adapted => }/group_norm.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/group_norm_impl.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/gru.hpp (67%) create mode 100644 src/mlpack/methods/ann/layer/gru_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/hard_tanh.hpp (71%) rename src/mlpack/methods/ann/layer/{not_adapted => }/hard_tanh_impl.hpp (71%) rename src/mlpack/methods/ann/layer/{not_adapted => }/hardshrink.hpp (67%) rename src/mlpack/methods/ann/layer/{not_adapted => }/hardshrink_impl.hpp (63%) create mode 100644 src/mlpack/methods/ann/layer/highway.hpp create mode 100644 src/mlpack/methods/ann/layer/highway_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/instance_norm.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/instance_norm_impl.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/isrlu.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/isrlu_impl.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/join.hpp (61%) rename src/mlpack/methods/ann/layer/{not_adapted => }/join_impl.hpp (63%) rename src/mlpack/methods/ann/layer/{not_adapted => }/layer_norm.hpp (67%) rename src/mlpack/methods/ann/layer/{not_adapted => }/layer_norm_impl.hpp (60%) create mode 100644 src/mlpack/methods/ann/layer/layer_traits.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/lookup.hpp (64%) rename src/mlpack/methods/ann/layer/{not_adapted => }/lookup_impl.hpp (58%) rename src/mlpack/methods/ann/layer/{not_adapted => }/lp_pooling.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/lp_pooling_impl.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/mean_pooling.hpp (73%) create mode 100644 src/mlpack/methods/ann/layer/mean_pooling_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/minibatch_discrimination.hpp (61%) create mode 100644 src/mlpack/methods/ann/layer/minibatch_discrimination_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/multi_layer.hpp delete mode 100644 src/mlpack/methods/ann/layer/multi_layer_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/multihead_attention.hpp (70%) rename src/mlpack/methods/ann/layer/{not_adapted => }/multihead_attention_impl.hpp (79%) rename src/mlpack/methods/ann/layer/{not_adapted => }/multiply_constant.hpp (64%) create mode 100644 src/mlpack/methods/ann/layer/multiply_constant_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/multiply_merge.hpp (51%) create mode 100644 src/mlpack/methods/ann/layer/multiply_merge_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/nearest_interpolation.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/nearest_interpolation_impl.hpp (100%) delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/README.md delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/add_merge_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/concat.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/concat_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/constant_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/gru_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/highway.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/highway_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/mean_pooling_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/minibatch_discrimination_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/multiply_constant_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/multiply_merge_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/recurrent_attention_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/recurrent_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/reparametrization_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/sequential.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/sequential_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/not_adapted/weight_norm_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/parametric_relu.hpp (64%) rename src/mlpack/methods/ann/layer/{not_adapted => }/parametric_relu_impl.hpp (51%) rename src/mlpack/methods/ann/layer/{not_adapted => }/pixel_shuffle.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/pixel_shuffle_impl.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/positional_encoding.hpp (63%) rename src/mlpack/methods/ann/layer/{not_adapted => }/positional_encoding_impl.hpp (59%) rename src/mlpack/methods/ann/layer/{not_adapted => }/recurrent.hpp (54%) rename src/mlpack/methods/ann/layer/{not_adapted => }/recurrent_attention.hpp (57%) create mode 100644 src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp create mode 100644 src/mlpack/methods/ann/layer/recurrent_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/recurrent_layer.hpp delete mode 100644 src/mlpack/methods/ann/layer/recurrent_layer_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/reinforce_normal.hpp (64%) rename src/mlpack/methods/ann/layer/{not_adapted => }/reinforce_normal_impl.hpp (68%) rename src/mlpack/methods/ann/layer/{not_adapted => }/relu6.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/relu6_impl.hpp (100%) rename src/mlpack/methods/ann/layer/{not_adapted => }/reparametrization.hpp (54%) create mode 100644 src/mlpack/methods/ann/layer/reparametrization_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/select.hpp (53%) rename src/mlpack/methods/ann/layer/{not_adapted => }/select_impl.hpp (58%) create mode 100644 src/mlpack/methods/ann/layer/sequential.hpp create mode 100644 src/mlpack/methods/ann/layer/sequential_impl.hpp delete mode 100644 src/mlpack/methods/ann/layer/serialization.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/softmax.hpp (58%) rename src/mlpack/methods/ann/layer/{not_adapted => }/softmax_impl.hpp (65%) rename src/mlpack/methods/ann/layer/{not_adapted => }/softmin.hpp (57%) rename src/mlpack/methods/ann/layer/{not_adapted => }/softmin_impl.hpp (65%) rename src/mlpack/methods/ann/layer/{not_adapted => }/softshrink.hpp (66%) rename src/mlpack/methods/ann/layer/{not_adapted => }/softshrink_impl.hpp (59%) rename src/mlpack/methods/ann/layer/{not_adapted => }/spatial_dropout.hpp (66%) rename src/mlpack/methods/ann/layer/{not_adapted => }/spatial_dropout_impl.hpp (57%) rename src/mlpack/methods/ann/layer/{not_adapted => }/subview.hpp (69%) rename src/mlpack/methods/ann/layer/{not_adapted => }/transposed_convolution.hpp (72%) rename src/mlpack/methods/ann/layer/{not_adapted => }/transposed_convolution_impl.hpp (79%) rename src/mlpack/methods/ann/layer/{not_adapted => }/virtual_batch_norm.hpp (60%) rename src/mlpack/methods/ann/layer/{not_adapted => }/virtual_batch_norm_impl.hpp (61%) rename src/mlpack/methods/ann/{loss_functions => layer}/vr_class_reward.hpp (58%) create mode 100644 src/mlpack/methods/ann/layer/vr_class_reward_impl.hpp rename src/mlpack/methods/ann/layer/{not_adapted => }/weight_norm.hpp (58%) create mode 100644 src/mlpack/methods/ann/layer/weight_norm_impl.hpp delete mode 100644 src/mlpack/methods/ann/loss_functions/vr_class_reward_impl.hpp delete mode 100644 src/mlpack/methods/ann/make_alias.hpp rename src/mlpack/methods/ann/{not_adapted => }/rbm/CMakeLists.txt (100%) rename src/mlpack/methods/ann/{not_adapted => }/rbm/rbm.hpp (100%) rename src/mlpack/methods/ann/{not_adapted => }/rbm/rbm_impl.hpp (100%) rename src/mlpack/methods/ann/{not_adapted => }/rbm/rbm_policies.hpp (100%) rename src/mlpack/methods/ann/{not_adapted => }/rbm/spike_slab_rbm_impl.hpp (100%) create mode 100644 src/mlpack/methods/ann/util/CMakeLists.txt create mode 100644 src/mlpack/methods/ann/util/check_input_shape.hpp create mode 100644 src/mlpack/methods/ann/visitor/CMakeLists.txt create mode 100644 src/mlpack/methods/ann/visitor/add_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/add_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/backward_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/backward_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/bias_set_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/copy_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/copy_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/delete_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/delete_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/delta_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/delta_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/deterministic_set_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/deterministic_set_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/forward_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/gradient_set_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/gradient_set_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/gradient_update_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/gradient_update_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/gradient_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/gradient_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/gradient_zero_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/gradient_zero_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/input_shape_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/input_shape_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/load_output_parameter_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/load_output_parameter_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/loss_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/output_height_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/output_height_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/output_parameter_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/output_parameter_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/output_width_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/output_width_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/parameters_set_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/parameters_set_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/parameters_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/parameters_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/reset_cell_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/reset_cell_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/reset_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/reset_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/reward_set_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/reward_set_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/run_set_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/run_set_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/save_output_parameter_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/save_output_parameter_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/set_input_height_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/set_input_height_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/set_input_width_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/set_input_width_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/weight_set_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/weight_set_visitor_impl.hpp create mode 100644 src/mlpack/methods/ann/visitor/weight_size_visitor.hpp create mode 100644 src/mlpack/methods/ann/visitor/weight_size_visitor_impl.hpp create mode 100644 src/mlpack/tests/ann_visitor_test.cpp rename src/mlpack/tests/{not_adapted => }/gan_test.cpp (100%) create mode 100644 src/mlpack/tests/layer_names_test.cpp rename src/mlpack/tests/{not_adapted => }/rbm_network_test.cpp (100%) rename src/mlpack/tests/{not_adapted => }/wgan_test.cpp (100%) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index a04e305bc3..5e87b118a5 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -80,7 +80,6 @@ steps: mkdir build && cd build if [ "$(binding)" == "go" ]; then export GOPATH=$PWD/src/mlpack/bindings/go - export GO111MODULE=off go get -u -t gonum.org/v1/gonum/... fi cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=`which python` -DCEREAL_INCLUDE_DIR=/usr/include/ .. diff --git a/.ci/macos-steps.yaml b/.ci/macos-steps.yaml index ef9a736fac..60f5e4d3b6 100644 --- a/.ci/macos-steps.yaml +++ b/.ci/macos-steps.yaml @@ -33,7 +33,6 @@ steps: mkdir build && cd build if [ "$(binding)" == "go" ]; then export GOPATH=$PWD/src/mlpack/bindings/go - export GO111MODULE=off go get -u -t gonum.org/v1/gonum/... fi if [ "$(binding)" == "python" ]; then diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8b6d7a7ffb..99e4039bfc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,7 +57,6 @@ jobs: run: | remotes::install_deps(dependencies = TRUE) remotes::install_cran("roxygen2") - remotes::install_cran("pkgbuild") shell: Rscript {0} - name: CMake diff --git a/CMake/FindGo.cmake b/CMake/FindGo.cmake index 6d5011f1c8..39b93f69aa 100644 --- a/CMake/FindGo.cmake +++ b/CMake/FindGo.cmake @@ -14,8 +14,8 @@ if (GO_EXECUTABLE) RESULT_VARIABLE RESULT ) if (RESULT EQUAL 0) - string(REGEX MATCH "([0-9]+\\.[0-9]+\(\\.[0-9]+\)?)" - GO_VERSION_STRING "${GO_VERSION_STRING}") + string(REGEX REPLACE ".*([0-9]+\\.[0-9]+\(\\.[0-9]+\)?).*" "\\1" + GO_VERSION_STRING ${GO_VERSION_STRING}) endif() endif() diff --git a/CMake/FindGonum.cmake b/CMake/FindGonum.cmake index 3667625915..eb18819a85 100644 --- a/CMake/FindGonum.cmake +++ b/CMake/FindGonum.cmake @@ -4,21 +4,20 @@ if (GO_EXECUTABLE) execute_process( COMMAND ${GO_EXECUTABLE} list gonum.org/v1/gonum/mat - OUTPUT_VARIABLE GONUM_RAW_STRING + OUTPUT_VARIABLE GONUM_VERSION_STRING RESULT_VARIABLE RESULT ) if (RESULT EQUAL 0) + string(REGEX REPLACE ".*([0-9]+\\.[0-9]+\\.[0-9]+[\n]+).*" "\\1" + GONUM_VERSION_STRING ${GONUM_VERSION_STRING}) string(REGEX REPLACE "\n$" "" - GONUM_RAW_STRING ${GONUM_RAW_STRING}) - if ("${GONUM_RAW_STRING}" STREQUAL "gonum.org/v1/gonum/mat") - set(GONUM_FOUND 1) - endif() + GONUM_VERSION_STRING ${GONUM_VERSION_STRING}) endif() endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args( Gonum - REQUIRED_VARS GONUM_FOUND + REQUIRED_VARS GONUM_VERSION_STRING FAIL_MESSAGE "Gonum not found" ) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 8bd3ffc06b..b9b43bdc6d 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -7,7 +7,7 @@ Source: Files: * Copyright: - Copyright 2008-2022, Ryan Curtin + Copyright 2008-2021, Ryan Curtin Copyright 2008-2013, Bill March Copyright 2008-2012, Dongryeol Lee Copyright 2008-2013, Nishant Mehta diff --git a/HISTORY.md b/HISTORY.md index 605aaec568..c5c0146a13 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,7 +1,5 @@ ### mlpack ?.?.? ###### ????-??-?? - * Fix `Perceptron` to work with cross-validation framework (#3190). - * Migrate from boost tests to Catch2 framework (#2523), (#2584). * Bump minimum armadillo version from 8.400 to 9.800 (#3043), (#3048). diff --git a/src/mlpack/bindings/python/copy_artifacts.py b/src/mlpack/bindings/python/copy_artifacts.py index 969be65451..ab62a403c5 100644 --- a/src/mlpack/bindings/python/copy_artifacts.py +++ b/src/mlpack/bindings/python/copy_artifacts.py @@ -6,14 +6,18 @@ # terms of the 3-clause BSD license. You should have received a copy of the # 3-clause BSD license along with mlpack. If not, see # http://www.opensource.org/licenses/BSD-3-Clause for more information. +import sys import sysconfig import shutil import os -import glob -# Match any lib.$platform*/mlpack/ directory. -directory = glob.glob('build/lib.' + sysconfig.get_platform() + '*/mlpack/')[0] -directory = directory.replace('\\', '/') +directory = 'build/lib.' + \ + sysconfig.get_platform() + \ + '-' + \ + str(sys.version_info[0]) + \ + '.' + \ + str(sys.version_info[1]) + \ + '/mlpack/' # Now copy all the files from the directory to the desired location. for f in os.listdir(directory): diff --git a/src/mlpack/bindings/python/is_serializable.hpp b/src/mlpack/bindings/python/is_serializable.hpp index f57553939a..dcec2986f5 100644 --- a/src/mlpack/bindings/python/is_serializable.hpp +++ b/src/mlpack/bindings/python/is_serializable.hpp @@ -21,7 +21,7 @@ namespace python { template inline bool IsSerializable( util::ParamData& /* d */, - const typename std::enable_if::value>::type* = 0) + const typename boost::disable_if>::type* = 0) { return false; } @@ -29,7 +29,7 @@ inline bool IsSerializable( template inline bool IsSerializable( util::ParamData& /* d */, - const typename std::enable_if::value>::type* = 0) + const typename boost::enable_if>::type* = 0) { return true; } diff --git a/src/mlpack/core/cereal/CMakeLists.txt b/src/mlpack/core/cereal/CMakeLists.txt index cad97c39ea..e6acb18e3f 100644 --- a/src/mlpack/core/cereal/CMakeLists.txt +++ b/src/mlpack/core/cereal/CMakeLists.txt @@ -7,6 +7,8 @@ set(SOURCES pair_associative_container.hpp pointer_wrapper.hpp pointer_vector_wrapper.hpp + pointer_variant_wrapper.hpp + pointer_vector_variant_wrapper.hpp unordered_map.hpp ) diff --git a/src/mlpack/core/cereal/pointer_variant_wrapper.hpp b/src/mlpack/core/cereal/pointer_variant_wrapper.hpp new file mode 100644 index 0000000000..92ac65f7fc --- /dev/null +++ b/src/mlpack/core/cereal/pointer_variant_wrapper.hpp @@ -0,0 +1,159 @@ +/** + * @file core/cereal/pointer_variant_wrapper.hpp + * @author Omar Shrit + * + * Implementation of a boost::variant wrapper to enable the serialization of + * the pointers inside boost variant in cereal + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CEREAL_POINTER_VARIANT_WRAPPER_HPP +#define MLPACK_CORE_CEREAL_POINTER_VARIANT_WRAPPER_HPP + +#include +#include +#include +#include + +#include +#include +#include + +#include "pointer_wrapper.hpp" + +namespace cereal { + +// Forward declaration. +template +class PointerVariantWrapper; + +/** + * Serialize a boost variant in which the variant it self is a raw pointer. + * This wrapper will wrap each variant independently by encapsulating each variant + * into the PoninterWrapper we have created already. + * + * @param t A reference to boost variant that holds raw pointer. + */ +template +inline PointerVariantWrapper +make_pointer_variant(boost::variant& t) +{ + return PointerVariantWrapper(t); +} + +template +struct save_visitor : public boost::static_visitor +{ + save_visitor(Archive& ar) : ar(ar) {} + + template + void operator()(const T* value) const + { + ar(CEREAL_POINTER(value)); + } + + template + void operator()(boost::variant& value) const + { + ar(make_pointer_variant(value)); + } + + Archive& ar; +}; + +template +struct load_visitor : public boost::static_visitor +{ + template + static void load_impl(Archive& ar, VariantType& variant, std::true_type) + { + // Note that T will be a pointer type. + T loadVariant; + ar(CEREAL_POINTER(loadVariant)); + variant = loadVariant; + } + + template + static void load_impl(Archive& ar, VariantType& value, std::false_type) + { + // This must be a nested boost::variant. + T loadVariant; + ar(make_pointer_variant(loadVariant)); + value = loadVariant; + } + + template + static void load(Archive& ar, VariantType& variant) + { + // Delegate to the proper load_impl() overload depending on whether T is a + // pointer type. If T is not a pointer type, then we expect it to be a + // nested boost::variant. + load_impl(ar, variant, typename std::is_pointer::type()); + } +}; + +/** + * The objective of this class is to create a wrapper for + * boost::variant. + * Cereal supports the serialization of boost::variant, but + * we need to serialize it if it holds a raw pointers. + * This class depeds on the PointerWrapper we have already created in which it is + * used to serialize each variant independently + */ +template +class PointerVariantWrapper +{ + public: + PointerVariantWrapper(boost::variant& pointerVar) : + pointerVariant(pointerVar) + {} + + template + void save(Archive& ar) const + { + // which represents the index in std::variant. + int which = pointerVariant.which(); + ar(CEREAL_NVP(which)); + save_visitor s(ar); + boost::apply_visitor(s, pointerVariant); + } + + template + void load(Archive& ar) + { + // Load the size of the serialized type. + int which; + ar(CEREAL_NVP(which)); + + // Create function pointers to each overload of load_visitor::load, for + // all T in VariantTypes. + using LoadFuncType = void(*)(Archive&, boost::variant&); + LoadFuncType loadFuncArray[] = { &load_visitor::load... }; + + if (which >= int(sizeof(loadFuncArray)/sizeof(loadFuncArray[0]))) + throw std::runtime_error("Invalid 'which' selector when" + "deserializing boost::variant"); + + loadFuncArray[which](ar, pointerVariant); + } + + private: + boost::variant& pointerVariant; +}; + +/** + * Cereal does not support the serialization of raw pointer. + * This macro enable developers to serialize boost::variant that holds raw + * pointers by using the above PointerVariantWrapper class which replace the + * internal raw pointers by smart pointer internally. + * + * @param T boost::variant that holds raw pointer to be serialized. + */ +#define CEREAL_VARIANT_POINTER(T) cereal::make_pointer_variant(T) + +} // namespace cereal + +#endif // CEREAL_POINTER_VARIANT_WRAPPER_HPP diff --git a/src/mlpack/core/cereal/pointer_vector_variant_wrapper.hpp b/src/mlpack/core/cereal/pointer_vector_variant_wrapper.hpp new file mode 100644 index 0000000000..76f035e7b5 --- /dev/null +++ b/src/mlpack/core/cereal/pointer_vector_variant_wrapper.hpp @@ -0,0 +1,97 @@ +/** + * @file core/cereal/pointer_vector_variant_wrapper.hpp + * @author Omar Shrit + * + * Implementation of a boost::variant wrapper to enable the serialization of + * the pointers inside boost variant in cereal + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CEREAL_POINTER_VECTOR_VARIANT_WRAPPER_HPP +#define MLPACK_CORE_CEREAL_POINTER_VECTOR_VARIANT_WRAPPER_HPP + +#include "pointer_wrapper.hpp" +#include "pointer_variant_wrapper.hpp" +#include "pointer_vector_wrapper.hpp" + +namespace cereal { + +// Forward declaration +template +class PointerVectorVariantWrapper; + +/** + * Serialize a std::vector of boost variants in which the variant in each boost + * variant is a raw pointer. + * This wrapper will wrap each boost variant independently by encapsulating each + * boost variant into the PoninterVariantWrapper we have created already. + * + * @param t A reference to a vector of boost variants that holds raw pointer. + */ +template +inline PointerVectorVariantWrapper +make_vector_pointer_variant(std::vector>& t) +{ + return PointerVectorVariantWrapper(t); +} + +/** + * The objective of this class is to create a wrapper for + * a vector of boost::variant that holds pointer. + * Cereal supports the serialization of boost::variant, but + * we need to serialize it if it holds a vector of boost::variant that holds a + * pointers. + */ +template +class PointerVectorVariantWrapper +{ + public: + PointerVectorVariantWrapper( + std::vector>& vecPointerVar) + : vectorPointerVariant(vecPointerVar) + {} + + template + void save(Archive& ar) const + { + size_t vecSize = vectorPointerVariant.size(); + ar(CEREAL_NVP(vecSize)); + for (size_t i = 0; i < vectorPointerVariant.size(); ++i) + { + ar(CEREAL_VARIANT_POINTER(vectorPointerVariant.at(i))); + } + } + + template + void load(Archive& ar) + { + size_t vecSize = 0; + ar(CEREAL_NVP(vecSize)); + vectorPointerVariant.resize(vecSize); + for (size_t i = 0; i < vectorPointerVariant.size(); ++i) + { + ar(CEREAL_VARIANT_POINTER(vectorPointerVariant.at(i))); + } + } + + private: + std::vector>& vectorPointerVariant; +}; + +/** + * Cereal does not support the serialization of raw pointer. + * This macro enable developers to serialize a std vector that holds boost::variants + * that holds raw pointers by using the above PointerVectorVariantWrapper class + * which replace the internal raw pointers by smart pointer internally. + * + * @param T std::vector that holds raw pointer to be serialized. + */ +#define CEREAL_VECTOR_VARIANT_POINTER(T) cereal::make_vector_pointer_variant(T) + +} // namespace cereal + +#endif // CEREAL_POINTER_VECTOR_VARIANT_WRAPPER_HPP + diff --git a/src/mlpack/core/util/size_checks.hpp b/src/mlpack/core/util/size_checks.hpp index f5125a591b..c2d60459b8 100644 --- a/src/mlpack/core/util/size_checks.hpp +++ b/src/mlpack/core/util/size_checks.hpp @@ -26,27 +26,18 @@ namespace util { * error generation. * @param addInfo Name to use for labels for precise error generation. Default * is "labels"; for example, "weights" could also be used. - * @param isDataTranspose Bool parameter which can be set true to transpose data - * before size-check. Default is false. - * @param isLabelTranspose Bool parameter which can be set true to transpose label - * before size-check. Default is false. */ template inline void CheckSameSizes(const DataType& data, const LabelsType& label, const std::string& callerDescription, - const std::string& addInfo = "labels", - const bool& isDataTranspose = false, - const bool& isLabelTranspose = false) -{ - const size_t dataPoints = (isDataTranspose == true) ? data.n_rows : data.n_cols; - const size_t labelPoints = (isLabelTranspose == true) ? label.n_rows : label.n_cols; - - if (dataPoints != labelPoints) + const std::string& addInfo = "labels") +{ + if (data.n_cols != label.n_cols) { std::ostringstream oss; - oss << callerDescription << ": number of points (" << dataPoints << ") " - << "does not match number of " << addInfo << " (" << labelPoints + oss << callerDescription << ": number of points (" << data.n_cols << ") " + << "does not match number of " << addInfo << " (" << label.n_cols << ")!" << std::endl; throw std::invalid_argument(oss.str()); } diff --git a/src/mlpack/methods/ann/CMakeLists.txt b/src/mlpack/methods/ann/CMakeLists.txt index fb1101ec11..8888113548 100644 --- a/src/mlpack/methods/ann/CMakeLists.txt +++ b/src/mlpack/methods/ann/CMakeLists.txt @@ -3,17 +3,24 @@ set(SOURCES ffn.hpp ffn_impl.hpp - forward_decls.hpp - make_alias.hpp rnn.hpp rnn_impl.hpp + brnn.hpp + brnn_impl.hpp + layer_names.hpp ) +add_subdirectory(visitor) +add_subdirectory(activation_functions) add_subdirectory(init_rules) add_subdirectory(layer) add_subdirectory(loss_functions) add_subdirectory(convolution_rules) +add_subdirectory(gan) +add_subdirectory(rbm) +add_subdirectory(augmented) add_subdirectory(regularizer) +add_subdirectory(util) # Add directory name to sources. set(DIR_SRCS) diff --git a/src/mlpack/methods/ann/not_adapted/brnn.hpp b/src/mlpack/methods/ann/brnn.hpp similarity index 99% rename from src/mlpack/methods/ann/not_adapted/brnn.hpp rename to src/mlpack/methods/ann/brnn.hpp index ac1e77f3dc..e3170474b4 100644 --- a/src/mlpack/methods/ann/not_adapted/brnn.hpp +++ b/src/mlpack/methods/ann/brnn.hpp @@ -24,6 +24,7 @@ #include "init_rules/network_init.hpp" #include #include +#include #include #include diff --git a/src/mlpack/methods/ann/not_adapted/brnn_impl.hpp b/src/mlpack/methods/ann/brnn_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/brnn_impl.hpp rename to src/mlpack/methods/ann/brnn_impl.hpp diff --git a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp index c01fd6d749..d0bf1cadfd 100644 --- a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp @@ -57,15 +57,9 @@ class NaiveConvolution const size_t dilationW = 1, const size_t dilationH = 1) { - // Compute the output size. The filterRows and filterCols computation must - // take into account the fact that dilation only adds rows or columns - // *between* filter elements. So, e.g., a dilation of 2 on a kernel size of - // 3x3 means an effective kernel size of 5x5, *not* 6x6. - const size_t filterRows = filter.n_rows * dilationH - (dilationH - 1); - const size_t filterCols = filter.n_cols * dilationW - (dilationW - 1); - const size_t outputRows = (input.n_rows - filterRows + dH) / dH; - const size_t outputCols = (input.n_cols - filterCols + dW) / dW; - output.zeros(outputRows, outputCols); + output = arma::zeros >( + (input.n_rows - (filter.n_rows - 1) * dilationW - 1) / dW + 1, + (input.n_cols - (filter.n_cols - 1) * dilationH - 1) / dH + 1); // It seems to be about 3.5 times faster to use pointers instead of // filter(ki, kj) * input(leftInput + ki, topInput + kj) and output(i, j). @@ -109,22 +103,37 @@ class NaiveConvolution const size_t dilationW = 1, const size_t dilationH = 1) { - // First, compute the necessary padding for the full convolution. It is - // possible that this might be an overestimate. Note that these variables - // only hold the padding on one side of the input. - const size_t filterRows = filter.n_rows * dilationH - (dilationH - 1); - const size_t filterCols = filter.n_cols * dilationW - (dilationW - 1); - const size_t paddingRows = filterRows - 1; - const size_t paddingCols = filterCols - 1; + size_t outputRows = (input.n_rows - 1) * dW + 2 * (filter.n_rows - 1) + * dilationW + 1; + size_t outputCols = (input.n_cols - 1) * dH + 2 * (filter.n_cols - 1) + * dilationH + 1; + + for (size_t i = 0; i < dW; ++i) + { + if (((((i + outputRows - 2 * (filter.n_rows - 1) * dilationW - 1) % dW) + + dW) % dW) == i){ + outputRows += i; + break; + } + } + for (size_t i = 0; i < dH; ++i) + { + if (((((i + outputCols - 2 * (filter.n_cols - 1) * dilationH - 1) % dH) + + dH) % dH) == i){ + outputCols += i; + break; + } + } // Pad filter and input to the working output shape. - arma::Mat inputPadded(input.n_rows + 2 * paddingRows, - input.n_cols + 2 * paddingCols, arma::fill::zeros); - inputPadded.submat(paddingRows, paddingCols, paddingRows + input.n_rows - 1, - paddingCols + input.n_cols - 1) = input; + arma::Mat inputPadded = arma::zeros >(outputRows, + outputCols); + inputPadded.submat((filter.n_rows - 1) * dilationW, (filter.n_cols - 1) + * dilationH, (filter.n_rows - 1) * dilationW + input.n_rows - 1, + (filter.n_cols - 1) * dilationH + input.n_cols - 1) = input; NaiveConvolution::Convolution(inputPadded, filter, - output, dW, dH, dilationW, dilationH); + output, 1, 1, dilationW, dilationH); } /* diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp index 0b6bde3648..1c65bbb749 100644 --- a/src/mlpack/methods/ann/ffn.hpp +++ b/src/mlpack/methods/ann/ffn.hpp @@ -15,49 +15,46 @@ #include -#include "forward_decls.hpp" +#include "visitor/delete_visitor.hpp" +#include "visitor/delta_visitor.hpp" +#include "visitor/output_height_visitor.hpp" +#include "visitor/output_parameter_visitor.hpp" +#include "visitor/output_width_visitor.hpp" +#include "visitor/reset_visitor.hpp" +#include "visitor/weight_size_visitor.hpp" +#include "visitor/copy_visitor.hpp" +#include "visitor/loss_visitor.hpp" + #include "init_rules/network_init.hpp" +#include #include -#include #include -#include +#include #include namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Implementation of a standard feed forward network. Any layer that inherits - * from the base `Layer` class can be added to this model. For recursive neural - * networks, see the `RNN` class. - * - * In general, a network can be created by using the `Add()` method to add - * layers to the network. Then, training can be performed with `Train()`, and - * data points can be passed through the trained network with `Predict()`. - * - * Although the actual types passed as input will be matrix objects with one - * data point per column, each data point can be a tensor of arbitrary shape. - * If data points are not 1-dimensional vectors, then set the shape of the input - * with `InputDimensions()` before calling `Train()`. - * - * More granular functionality is available with `Forward()`, Backward()`, and - * `Evaluate()`, or even by accessing the individual layers directly with - * `Network()`. + * Implementation of a standard feed forward network. * * @tparam OutputLayerType The output layer type used to evaluate the network. * @tparam InitializationRuleType Rule used to initialize the weight matrix. - * @tparam MatType Type of matrix to be given as input to the network. - * @tparam MatType Type of matrix to be produced as output from the last - * layer. + * @tparam CustomLayers Any set of custom layers that could be a part of the + * feed forward network. */ template< - typename OutputLayerType = NegativeLogLikelihood, - typename InitializationRuleType = RandomInitialization, - typename MatType = arma::mat> + typename OutputLayerType = NegativeLogLikelihood<>, + typename InitializationRuleType = RandomInitialization, + typename... CustomLayers +> class FFN { public: + //! Convenience typedef for the internal model construction. + using NetworkType = FFN; + /** * Create the FFN object. * @@ -75,73 +72,56 @@ class FFN InitializationRuleType initializeRule = InitializationRuleType()); //! Copy constructor. - FFN(const FFN& other); + FFN(const FFN&); + //! Move constructor. - FFN(FFN&& other); - //! Copy operator. - FFN& operator=(const FFN& other); - //! Move assignment operator. - FFN& operator=(FFN&& other); + FFN(FFN&&); + + //! Copy/move assignment operator. + FFN& operator = (FFN); + + //! Destructor to release allocated memory. + ~FFN(); /** - * Add a new layer to the model. + * Check if the optimizer has MaxIterations() parameter, if it does + * then check if it's value is less than the number of datapoints + * in the dataset. * - * @param args The layer parameter. + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. */ - template - void Add(Args... args) - { - network.template Add(args...); - inputDimensionsAreSet = false; - } + template + typename std::enable_if< + HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; /** - * Add a new layer to the model. Note that any trainable weights of this - * layer will be reset! (Any constant parameters are kept.) + * Check if the optimizer has MaxIterations() parameter, if it + * doesn't then simply return from the function. * - * @param layer The Layer to be added to the model. + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. */ - void Add(Layer* layer) - { - network.Add(layer); - inputDimensionsAreSet = false; - } - - //! Get the layers of the network. - const std::vector*>& Network() const - { - return network.Network(); - } - - /** - * Modify the network model. Be careful! If you change the structure of the - * network or parameters for layers, its state may become invalid, and the - * next time it is used for any operation the parameters will be reset. - * - * Don't add any layers like this; use `Add()` instead. - */ - std::vector*>& Network() - { - // We can no longer make any assumptions... the user may change anything. - inputDimensionsAreSet = false; - layerMemoryIsSet = false; - - return network.Network(); - } + template + typename std::enable_if< + !HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; /** * Train the feedforward network on the given input data using the given * optimizer. * - * If no parameters have ever been set (e.g. if `Parameters()` is an empty - * matrix), or if the parameters' size does not match the number of weights - * needed for the current input size (as given by `predictors` and optionally - * set further by `InputDimensions()`), then the network will be initialized - * using `InitializeRuleType`. + * This will use the existing model parameters as a starting point for the + * optimization. If this is not what you want, then you should access the + * parameters vector directly with Parameters() and modify it as desired. * - * If parameters are the right size for the given `predictors` and - * `InputDimensions()`, then the existing parameters will be used as a - * starting point. (If you want to reinitialize, first call `Reset()`.) + * If you want to pass in a parameter and discard the original parameter + * object, be sure to use std::move to avoid unnecessary copy. * * @tparam OptimizerType Type of optimizer to use to train the model. * @tparam CallbackTypes Types of Callback Functions. @@ -153,25 +133,22 @@ class FFN * @return The final objective of the trained model (NaN or Inf on error). */ template - typename MatType::elem_type Train(MatType predictors, - MatType responses, - OptimizerType& optimizer, - CallbackTypes&&... callbacks); + double Train(arma::mat predictors, + arma::mat responses, + OptimizerType& optimizer, + CallbackTypes&&... callbacks); /** * Train the feedforward network on the given input data. By default, the * RMSProp optimization algorithm is used, but others can be specified * (such as ens::SGD). * - * If no parameters have ever been set (e.g. if `Parameters()` is an empty - * matrix), or if the parameters' size does not match the number of weights - * needed for the current input size (as given by `predictors` and optionally - * set further by `InputDimensions()`), then the network will be initialized - * using `InitializeRuleType`. + * This will use the existing model parameters as a starting point for the + * optimization. If this is not what you want, then you should access the + * parameters vector directly with Parameters() and modify it as desired. * - * If parameters are the right size for the given `predictors` and - * `InputDimensions()`, then the existing parameters will be used as a - * starting point. (If you want to reinitialize, first call `Reset()`.) + * If you want to pass in a parameter and discard the original parameter + * object, be sure to use std::move to avoid unnecessary copy. * * @tparam OptimizerType Type of optimizer to use to train the model. * @param predictors Input training variables. @@ -182,123 +159,22 @@ class FFN * @return The final objective of the trained model (NaN or Inf on error). */ template - typename MatType::elem_type Train(MatType predictors, - MatType responses, - CallbackTypes&&... callbacks); + double Train(arma::mat predictors, + arma::mat responses, + CallbackTypes&&... callbacks); /** - * Predict the responses to a given set of predictors. The responses will be - * the output of the output layer when `predictors` is passed through the - * whole network (`OutputLayerType`). + * Predict the responses to a given set of predictors. The responses will + * reflect the output of the given output layer as returned by the + * output layer function. + * + * If you want to pass in a parameter and discard the original parameter + * object, be sure to use std::move to avoid unnecessary copy. * * @param predictors Input predictors. * @param results Matrix to put output predictions of responses into. - * @param batchSize Batch size to use for prediction. */ - void Predict(MatType predictors, - MatType& results, - const size_t batchSize = 128); - - // Return the number of weights in the model. - size_t WeightSize(); - - /** - * Set the logical dimensions of the input. `Train()` and `Predict()` expect - * data to be passed such that one point corresponds to one column, but this - * data is allowed to be an arbitrary higher-order tensor. - * - * So, if the input is meant to be 28x28x3 images, then the - * input data to `Train()` or `Predict()` should have 28*28*3 = 2352 rows, and - * `InputDimensions()` should be set to `{ 28, 28, 3 }`. Then, the layers of - * the network will interpret each input point as a 3-dimensional image - * instead of a 1-dimensional vector. - * - * If `InputDimensions()` is left unset before training, the data will be - * assumed to be a 1-dimensional vector. - */ - std::vector& InputDimensions() - { - // The user may change the input dimensions, so we will have to propagate - // these changes to the network. - inputDimensionsAreSet = false; - return inputDimensions; - } - //! Get the logical dimensions of the input. - const std::vector& InputDimensions() const { return inputDimensions; } - - //! Return the current set of weights. These are linearized: this contains - //! the weights of every layer. - const MatType& Parameters() const { return parameters; } - //! Modify the current set of weights. These are linearized: this contains - //! the weights of every layer. Be careful! If you change the shape of - //! `parameters` to something incorrect, it may be re-initialized the next - //! time a forward pass is done. - MatType& Parameters() { return parameters; } - - /** - * Reset the stored data of the network entirely. This resets all weights of - * each layer using `InitializationRuleType`, and prepares the network to - * accept a (flat 1-d) input size of `inputDimensionality` (if passed), or - * whatever input size has been set with `InputDimensions()`. - * - * This also resets the mode of the network to prediction mode (not training - * mode). See `SetNetworkMode()` for more information. - */ - void Reset(const size_t inputDimensionality = 0); - - /** - * Set all the layers in the network to training mode, if `training` is - * `true`, or set all the layers in the network to testing mode, if `training` - * is `false`. - */ - void SetNetworkMode(const bool training); - - /** - * Perform a manual forward pass of the data. - * - * `Forward()` and `Backward()` should be used as a pair, and they are - * designed mainly for advanced users. You should try to use `Predict()` and - * `Train()`, if you can. - * - * @param inputs The input data. - * @param results The predicted results. - */ - void Forward(const MatType& inputs, MatType& results); - - /** - * Perform a manual partial forward pass of the data. - * - * This function is meant for the cases when users require a forward pass only - * through certain layers and not the entire network. `Forward()` and - * `Backward()` should be used as a pair, and they are designed mainly for - * advanced users. You should try to use `Predict()` and `Train()`, if you - * can. - * - * @param inputs The input data for the specified first layer. - * @param results The predicted results from the specified last layer. - * @param begin The index of the first layer. - * @param end The index of the last layer. - */ - void Forward(const MatType& inputs, - MatType& results, - const size_t begin, - const size_t end); - - /** - * Perform a manual backward pass of the data. - * - * `Forward()` and `Backward()` should be used as a pair, and they are - * designed mainly for advanced users. You should try to use `Predict()` and - * `Train()` instead, if you can. - * - * @param inputs Inputs of current pass. - * @param targets The training target. - * @param gradients Computed gradients. - * @return Training error of the current pass. - */ - typename MatType::elem_type Backward(const MatType& inputs, - const MatType& targets, - MatType& gradients); + void Predict(arma::mat predictors, arma::mat& results); /** * Evaluate the feedforward network with the given predictors and responses. @@ -307,38 +183,41 @@ class FFN * @param predictors Input variables. * @param responses Target outputs for input variables. */ - typename MatType::elem_type Evaluate(const MatType& predictors, - const MatType& responses); - - //! Serialize the model. - template - void serialize(Archive& ar, const uint32_t /* version */); - - // - // Only ensmallen utility functions for training are found below here. - // They aren't generally useful otherwise. - // + template + double Evaluate(const PredictorsType& predictors, + const ResponsesType& responses); /** - * Note: this function is implemented so that it can be used by ensmallen's - * optimizers. It's not generally meant to be used otherwise. - * - * Evaluate the feedforward network with the given parameters. + * Evaluate the feedforward network with the given parameters. This function + * is usually called by the optimizer to train the model. * * @param parameters Matrix model parameters. */ - typename MatType::elem_type Evaluate(const MatType& parameters); + double Evaluate(const arma::mat& parameters); - /** - * Note: this function is implemented so that it can be used by ensmallen's - * optimizers. It's not generally meant to be used otherwise. - * + /** * Evaluate the feedforward network with the given parameters, but using only * a number of data points. This is useful for optimizers such as SGD, which * require a separable objective function. * - * Note that the network may return different results depending on the mode it - * is in (see `SetNetworkMode()`). + * @param parameters Matrix model parameters. + * @param begin Index of the starting point to use for objective function + * evaluation. + * @param batchSize Number of points to be passed at a time to use for + * objective function evaluation. + * @param deterministic Whether or not to train or test the model. Note some + * layer act differently in training or testing mode. + */ + double Evaluate(const arma::mat& parameters, + const size_t begin, + const size_t batchSize, + const bool deterministic); + + /** + * Evaluate the feedforward network with the given parameters, but using only + * a number of data points. This is useful for optimizers such as SGD, which + * require a separable objective function. This just calls the overload of + * Evaluate() with deterministic = true. * * @param parameters Matrix model parameters. * @param begin Index of the starting point to use for objective function @@ -346,14 +225,11 @@ class FFN * @param batchSize Number of points to be passed at a time to use for * objective function evaluation. */ - typename MatType::elem_type Evaluate(const MatType& parameters, - const size_t begin, - const size_t batchSize); + double Evaluate(const arma::mat& parameters, + const size_t begin, + const size_t batchSize); /** - * Note: this function is implemented so that it can be used by ensmallen's - * optimizers. It's not generally meant to be used otherwise. - * * Evaluate the feedforward network with the given parameters. * This function is usually called by the optimizer to train the model. * This just calls the overload of EvaluateWithGradient() with batchSize = 1. @@ -361,13 +237,10 @@ class FFN * @param parameters Matrix model parameters. * @param gradient Matrix to output gradient into. */ - typename MatType::elem_type EvaluateWithGradient(const MatType& parameters, - MatType& gradient); + template + double EvaluateWithGradient(const arma::mat& parameters, GradType& gradient); - /** - * Note: this function is implemented so that it can be used by ensmallen's - * optimizers. It's not generally meant to be used otherwise. - * + /** * Evaluate the feedforward network with the given parameters, but using only * a number of data points. This is useful for optimizers such as SGD, which * require a separable objective function. @@ -379,15 +252,13 @@ class FFN * @param batchSize Number of points to be passed at a time to use for * objective function evaluation. */ - typename MatType::elem_type EvaluateWithGradient(const MatType& parameters, - const size_t begin, - MatType& gradient, - const size_t batchSize); + template + double EvaluateWithGradient(const arma::mat& parameters, + const size_t begin, + GradType& gradient, + const size_t batchSize); /** - * Note: this function is implemented so that it can be used by ensmallen's - * optimizers. It's not generally meant to be used otherwise. - * * Evaluate the gradient of the feedforward network with the given parameters, * and with respect to only a number of points in the dataset. This is useful * for optimizers such as SGD, which require a separable objective function. @@ -399,156 +270,253 @@ class FFN * @param batchSize Number of points to be processed as a batch for objective * function gradient evaluation. */ - void Gradient(const MatType& parameters, + void Gradient(const arma::mat& parameters, const size_t begin, - MatType& gradient, + arma::mat& gradient, const size_t batchSize); /** - * Note: this function is implemented so that it can be used by ensmallen's - * optimizers. It's not generally meant to be used otherwise. - * - * Return the number of separable functions (the number of predictor points). - */ - size_t NumFunctions() const { return responses.n_cols; } - - /** - * Note: this function is implemented so that it can be used by ensmallen's - * optimizers. It's not generally meant to be used otherwise. - * - * Shuffle the order of function visitation. (This is equivalent to shuffling - * the dataset during training.) + * Shuffle the order of function visitation. This may be called by the + * optimizer. */ void Shuffle(); - /** - * Prepare the network for training on the given data. + /* + * Add a new module to the model. * - * This function won't actually trigger the training process, and is - * generally only useful internally. + * @param args The layer parameter. + */ + template + void Add(Args... args) { network.push_back(new LayerType(args...)); } + + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + + //! Get the network model. + const std::vector >& Model() const + { + return network; + } + //! Modify the network model. Be careful! If you change the structure of the + //! network or parameters for layers, its state may become invalid, so be sure + //! to call ResetParameters() afterwards. + std::vector >& Model() { return network; } + + //! Return the number of separable functions (the number of predictor points). + size_t NumFunctions() const { return numFunctions; } + + //! Return the initial point for the optimization. + const arma::mat& Parameters() const { return parameter; } + //! Modify the initial point for the optimization. + arma::mat& Parameters() { return parameter; } + + //! Get the matrix of responses to the input data points. + const arma::mat& Responses() const { return responses; } + //! Modify the matrix of responses to the input data points. + arma::mat& Responses() { return responses; } + + //! Get the matrix of data points (predictors). + const arma::mat& Predictors() const { return predictors; } + //! Modify the matrix of data points (predictors). + arma::mat& Predictors() { return predictors; } + + /** + * Reset the module infomration (weights/parameters). + */ + void ResetParameters(); + + //! Serialize the model. + template + void serialize(Archive& ar, const uint32_t /* version */); + + /** + * Perform the forward pass of the data in real batch mode. + * + * Forward and Backward should be used as a pair, and they are designed mainly + * for advanced users. User should try to use Predict and Train unless those + * two functions can't satisfy some special requirements. + * + * @param inputs The input data. + * @param results The predicted results. + */ + template + void Forward(const PredictorsType& inputs, ResponsesType& results); + + /** + * Perform a partial forward pass of the data. + * + * This function is meant for the cases when users require a forward pass only + * through certain layers and not the entire network. + * + * @param inputs The input data for the specified first layer. + * @param results The predicted results from the specified last layer. + * @param begin The index of the first layer. + * @param end The index of the last layer. + */ + template + void Forward(const PredictorsType& inputs , + ResponsesType& results, + const size_t begin, + const size_t end); + + /** + * Perform the backward pass of the data in real batch mode. + * + * Forward and Backward should be used as a pair, and they are designed mainly + * for advanced users. User should try to use Predict and Train unless those + * two functions can't satisfy some special requirements. + * + * @param inputs Inputs of current pass. + * @param targets The training target. + * @param gradients Computed gradients. + * @return Training error of the current pass. + */ + template + double Backward(const PredictorsType& inputs, + const TargetsType& targets, + GradientsType& gradients); + + private: + // Helper functions. + /** + * The Forward algorithm (part of the Forward-Backward algorithm). Computes + * forward probabilities for each module. + * + * @param input Data sequence to compute probabilities for. + */ + template + void Forward(const InputType& input); + + /** + * Prepare the network for the given data. + * This function won't actually trigger training process. * * @param predictors Input data variables. * @param responses Outputs results from input data variables. */ - void ResetData(MatType predictors, MatType responses); - - private: - // Helper functions. - - //! Use the InitializationPolicy to initialize all the weights in the network. - void InitializeWeights(); - - //! Make the memory of each layer point to the right place, by calling - //! SetWeightPtr() on each layer. - void SetLayerMemory(); + void ResetData(arma::mat predictors, arma::mat responses); /** - * Ensure that all the locally-cached information about the network is valid, - * all parameter memory is initialized, and we can make forward and backward - * passes. + * The Backward algorithm (part of the Forward-Backward algorithm). Computes + * backward pass for module. + */ + void Backward(); + + /** + * Iterate through all layer modules and update the the gradient using the + * layer defined optimizer. + */ + template + void Gradient(const InputType& input); + + /** + * Reset the module status by setting the current deterministic parameter + * for all modules that implement the Deterministic function. + */ + void ResetDeterministic(); + + /** + * Reset the gradient for all modules that implement the Gradient function. + */ + void ResetGradients(arma::mat& gradient); + + /** + * Swap the content of this network with given network. * - * @param functionName Name of function to use if an exception is thrown. - * @param inputDimensionality Given dimensionality of the input data. - * @param setMode If true, the mode of the network will be set to the - * parameter given in `training`. Otherwise the mode of the network is - * left unmodified. - * @param training Mode to set the network to; `true` indicates the network - * should be set to training mode; `false` indicates testing mode. + * @param network Desired source network. */ - void CheckNetwork(const std::string& functionName, - const size_t inputDimensionality, - const bool setMode = false, - const bool training = false); + void Swap(FFN& network); - /** - * Set the input and output dimensions of each layer in the network correctly. - * The size of the input is taken, in case `inputDimensions` has not been set - * otherwise (e.g. via `InputDimensions()`). If `InputDimensions()` is not - * empty, then `inputDimensionality` is ignored. - */ - void UpdateDimensions(const std::string& functionName, - const size_t inputDimensionality = 0); - - /** - * Check if the optimizer has MaxIterations() parameter, if it does then check - * if its value is less than the number of datapoints in the dataset. - * - * @tparam OptimizerType Type of optimizer to use to train the model. - * @param optimizer optimizer used in the training process. - * @param samples Number of datapoints in the dataset. - */ - template - typename std::enable_if< - ens::traits::HasMaxIterationsSignature::value, void - >::type - WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; - - /** - * Check if the optimizer has MaxIterations() parameter; if it doesn't then - * simply return from the function. - * - * @tparam OptimizerType Type of optimizer to use to train the model. - * @param optimizer optimizer used in the training process. - * @param samples Number of datapoints in the dataset. - */ - template - typename std::enable_if< - !ens::traits::HasMaxIterationsSignature::value, void - >::type - WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; - - //! Instantiated output layer used to evaluate the network. + //! Instantiated outputlayer used to evaluate the network. OutputLayerType outputLayer; //! Instantiated InitializationRule object for initializing the network //! parameter. InitializationRuleType initializeRule; - //! All of the network is stored inside this multilayer. - MultiLayer network; + //! The input width. + size_t width; - /** - * Matrix of (trainable) parameters. Each weight here corresponds to a layer, - * and each layer's `parameters` member is an alias pointing to parameters in - * this matrix. - * - * Note: although each layer may have its own MatType and MatType, - * ensmallen optimization requires everything to be stored in one matrix - * object, so we have chosen MatType. This could be made more flexible - * with a "wrapper" class implementing the Armadillo API. - */ - MatType parameters; + //! The input height. + size_t height; - //! Dimensions of input data. - std::vector inputDimensions; + //! Indicator if we already trained the model. + bool reset; - //! The matrix of data points (predictors). This member is empty, except - //! during training---we must store a local copy of the training data since - //! the ensmallen optimizer will not provide training data. - MatType predictors; + //! Locally-stored model modules. + std::vector > network; - //! The matrix of responses to the input data points. This member is empty, - //! except during training. - MatType responses; + //! The matrix of data points (predictors). + arma::mat predictors; - //! Locally-stored output of the network from a forward pass; used by the - //! backward pass. - MatType networkOutput; - //! Locally-stored output of the backward pass; used by the gradient pass. - MatType networkDelta; - //! Locally-stored error of the backward pass; used by the gradient pass. - MatType error; + //! The matrix of responses to the input data points. + arma::mat responses; - //! If true, each layer has its memory properly set for a forward/backward - //! pass. - bool layerMemoryIsSet; + //! Matrix of (trained) parameters. + arma::mat parameter; - //! If true, each layer has its inputDimensions properly set, and - //! `totalInputSize` and `totalOutputSize` are valid. - bool inputDimensionsAreSet; + //! The number of separable functions (the number of predictor points). + size_t numFunctions; - // RNN will call `CheckNetwork()`, which is private. - friend class RNN; + //! The current error for the backward pass. + arma::mat error; + + //! Locally-stored delta visitor. + DeltaVisitor deltaVisitor; + + //! Locally-stored output parameter visitor. + OutputParameterVisitor outputParameterVisitor; + + //! Locally-stored weight size visitor. + WeightSizeVisitor weightSizeVisitor; + + //! Locally-stored output width visitor. + OutputWidthVisitor outputWidthVisitor; + + //! Locally-stored output height visitor. + OutputHeightVisitor outputHeightVisitor; + + //! Locally-stored loss visitor + LossVisitor lossVisitor; + + //! Locally-stored reset visitor. + ResetVisitor resetVisitor; + + //! Locally-stored delete visitor. + DeleteVisitor deleteVisitor; + + //! The current evaluation mode (training or testing). + bool deterministic; + + //! Locally-stored delta object. + arma::mat delta; + + //! Locally-stored input parameter object. + arma::mat inputParameter; + + //! Locally-stored output parameter object. + arma::mat outputParameter; + + //! Locally-stored gradient parameter. + arma::mat gradient; + + //! Locally-stored copy visitor + CopyVisitor copyVisitor; + + // The GAN class should have access to internal members. + template< + typename Model, + typename InitializerType, + typename NoiseType, + typename PolicyType + > + friend class GAN; }; // class FFN } // namespace ann diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index 734b961301..c30573ea9b 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -15,694 +15,662 @@ // In case it hasn't been included yet. #include "ffn.hpp" -#include "make_alias.hpp" +#include "visitor/forward_visitor.hpp" +#include "visitor/backward_visitor.hpp" +#include "visitor/deterministic_set_visitor.hpp" +#include "visitor/gradient_set_visitor.hpp" +#include "visitor/gradient_visitor.hpp" +#include "visitor/set_input_height_visitor.hpp" +#include "visitor/set_input_width_visitor.hpp" + +#include "util/check_input_shape.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::FFN(OutputLayerType outputLayer, InitializationRuleType initializeRule) : + +template +FFN::FFN( + OutputLayerType outputLayer, InitializationRuleType initializeRule) : outputLayer(std::move(outputLayer)), initializeRule(std::move(initializeRule)), - layerMemoryIsSet(false), - inputDimensionsAreSet(false) + width(0), + height(0), + reset(false), + numFunctions(0), + deterministic(false) { /* Nothing to do here. */ } -template -FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::FFN(const FFN& network): - outputLayer(network.outputLayer), - initializeRule(network.initializeRule), - network(network.network), - parameters(network.parameters), - inputDimensions(network.inputDimensions), - predictors(network.predictors), - responses(network.responses), - // These will be set correctly in the first Forward() call. - layerMemoryIsSet(false), - inputDimensionsAreSet(false) +template +FFN::~FFN() { - // Nothing to do. -}; - -template -FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::FFN(FFN&& network): - outputLayer(std::move(network.outputLayer)), - initializeRule(std::move(network.initializeRule)), - network(std::move(network.network)), - parameters(std::move(network.parameters)), - inputDimensions(std::move(network.inputDimensions)), - predictors(std::move(network.predictors)), - responses(std::move(network.responses)), - // Aliases will not be correct after a std::move(), so we will manually - // reset them. - layerMemoryIsSet(false), - inputDimensionsAreSet(std::move(network.inputDimensionsAreSet)) -{ - // Nothing to do. -}; - -template -FFN& FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::operator=(const FFN& other) -{ - if (this != &other) - { - outputLayer = other.outputLayer; - initializeRule = other.initializeRule; - network = other.network; - parameters = other.parameters; - inputDimensions = other.inputDimensions; - predictors = other.predictors; - responses = other.responses; - networkOutput = other.networkOutput; - networkDelta = other.networkDelta; - error = other.error; - inputDimensionsAreSet = other.inputDimensionsAreSet; - - // Copying will not preserve Armadillo aliases correctly, so we will reset - // those. - layerMemoryIsSet = false; - } - - return *this; + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deleteVisitor)); } -template -FFN& FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::operator=(FFN&& other) -{ - if (this != &other) - { - outputLayer = std::move(other.outputLayer); - initializeRule = std::move(other.initializeRule); - network = std::move(other.network); - parameters = std::move(other.parameters); - inputDimensions = std::move(other.inputDimensions); - predictors = std::move(other.predictors); - responses = std::move(other.responses); - networkOutput = std::move(other.networkOutput); - networkDelta = std::move(other.networkDelta); - error = std::move(other.error); - inputDimensionsAreSet = std::move(other.inputDimensionsAreSet); - layerMemoryIsSet = std::move(other.layerMemoryIsSet); - } - - return *this; -} - -template -template -typename MatType::elem_type FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Train(MatType predictors, - MatType responses, - OptimizerType& optimizer, - CallbackTypes&&... callbacks) -{ - ResetData(std::move(predictors), std::move(responses)); - - WarnMessageMaxIterations(optimizer, this->predictors.n_cols); - - // Ensure that the network can be used. - CheckNetwork("FFN::Train()", this->predictors.n_rows, true, true); - - // Train the model. - Timer::Start("ffn_optimization"); - const typename MatType::elem_type out = - optimizer.Optimize(*this, parameters, callbacks...); - Timer::Stop("ffn_optimization"); - - Log::Info << "FFN::Train(): final objective of trained model is " << out - << "." << std::endl; - return out; -} - -template -template -typename MatType::elem_type FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Train(MatType predictors, - MatType responses, - CallbackTypes&&... callbacks) -{ - OptimizerType optimizer; - return Train(std::move(predictors), std::move(responses), optimizer, - callbacks...); -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Predict(MatType predictors, MatType& results, const size_t batchSize) -{ - // Ensure that the network is configured correctly. - CheckNetwork("FFN::Predict()", predictors.n_rows, true, false); - - results.set_size(network.OutputSize(), predictors.n_cols); - - for (size_t i = 0; i < predictors.n_cols; i += batchSize) - { - const size_t effectiveBatchSize = std::min(batchSize, - size_t(predictors.n_cols) - i); - - MatType predictorAlias(predictors.colptr(i), predictors.n_rows, - effectiveBatchSize, false, true); - MatType resultAlias(results.colptr(i), results.n_rows, - effectiveBatchSize, false, true); - - network.Forward(predictorAlias, resultAlias); - } -} - -template -size_t FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::WeightSize() -{ - // If the input dimensions have not yet been propagated to the network, we - // must do that now. - UpdateDimensions("FFN::WeightSize()"); - return network.WeightSize(); -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Reset(const size_t inputDimensionality) -{ - parameters.clear(); - - // If the user provided an input dimensionality, then we will take that as the - // new input size. Otherwise, whatever is currently specified in - // `InputDimensions()` will be used. - if (inputDimensionality != 0) - { - CheckNetwork("FFN::Reset()", inputDimensionality, true, false); - } - else - { - const size_t inputDims = std::accumulate(inputDimensions.begin(), - inputDimensions.end(), 0); - CheckNetwork("FFN::Reset()", inputDims, true, false); - } -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::SetNetworkMode(const bool training) -{ - network.Training() = training; -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Forward(const MatType& inputs, MatType& results) -{ - Forward(inputs, results, 0, network.Network().size() - 1); -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Forward(const MatType& inputs, - MatType& results, - const size_t begin, - const size_t end) -{ - // Sanity checking... - if (end < begin) - return; - - // Ensure the network is valid. - CheckNetwork("FFN::Forward()", inputs.n_rows); - - // We must always store a copy of the forward pass in `networkOutputs` in case - // we do a backward pass. - networkOutput.set_size(network.OutputSize(), inputs.n_cols); - network.Forward(inputs, networkOutput, begin, end); - - // It's possible the user passed `networkOutput` as `results`; in this case, - // we don't need to create an alias. - if (&results != &networkOutput) - results = networkOutput; -} - -template -typename MatType::elem_type FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Backward(const MatType& inputs, - const MatType& targets, - MatType& gradients) -{ - const typename MatType::elem_type res = - outputLayer.Forward(networkOutput, targets) + network.Loss(); - - // Compute the error of the output layer. - outputLayer.Backward(networkOutput, targets, error); - - // Perform the backward pass. - network.Backward(networkOutput, error, networkDelta); - - // Now compute the gradients. - // The gradient should have the same size as the parameters. - gradients.set_size(parameters.n_rows, parameters.n_cols); - network.Gradient(inputs, error, gradients); - - return res; -} - -template -typename MatType::elem_type FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Evaluate(const MatType& predictors, const MatType& responses) -{ - // Sanity check: ensure network is valid. - CheckNetwork("FFN::Evaluate()", predictors.n_rows); - - // Set networkOutput to the right size if needed, then perform the forward - // pass. - network.Forward(predictors, networkOutput); - - return outputLayer.Forward(networkOutput, responses) + network.Loss(); -} - -template -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::serialize(Archive& ar, const uint32_t /* version */) -{ - // Serialize the output layer and initialization rule. - ar(CEREAL_NVP(outputLayer)); - ar(CEREAL_NVP(initializeRule)); - - // Serialize the network itself. - ar(CEREAL_NVP(network)); - ar(CEREAL_NVP(parameters)); - - // Serialize the expected input size. - ar(CEREAL_NVP(inputDimensions)); - - // If we are loading, we need to initialize the weights. - if (cereal::is_loading()) - { - // We can clear these members, since it's not possible to serialize in the - // middle of training and resume. - predictors.clear(); - responses.clear(); - - networkOutput.clear(); - networkDelta.clear(); - - layerMemoryIsSet = false; - inputDimensionsAreSet = false; - - // The weights in `parameters` will be correctly set for each layer in the - // first call to Forward(). - } -} - -template -typename MatType::elem_type FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Evaluate(const MatType& parameters) -{ - typename MatType::elem_type res = 0; - for (size_t i = 0; i < predictors.n_cols; ++i) - res += Evaluate(parameters, i, 1); - - return res; -} - -template -typename MatType::elem_type FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Evaluate(const MatType& /* parameters */, - const size_t begin, - const size_t batchSize) -{ - CheckNetwork("FFN::Evaluate()", predictors.n_rows); - - // Set networkOutput to the right size if needed, then perform the forward - // pass. - networkOutput.set_size(network.OutputSize(), batchSize); - network.Forward(predictors.cols(begin, begin + batchSize - 1), networkOutput); - - return outputLayer.Forward(networkOutput, - responses.cols(begin, begin + batchSize - 1)) + network.Loss(); -} - -template -typename MatType::elem_type FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::EvaluateWithGradient(const MatType& parameters, MatType& gradient) -{ - typename MatType::elem_type res = 0; - res += EvaluateWithGradient(parameters, 0, gradient, 1); - for (size_t i = 1; i < predictors.n_cols; ++i) - { - arma::mat tmpGradient(gradient.n_rows, gradient.n_cols); - res += EvaluateWithGradient(parameters, i, tmpGradient, 1); - gradient += tmpGradient; - } - - return res; -} - -template -typename MatType::elem_type FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::EvaluateWithGradient(const MatType& parameters, - const size_t begin, - MatType& gradient, - const size_t batchSize) -{ - CheckNetwork("FFN::EvaluateWithGradient()", predictors.n_rows); - - // Set networkOutput to the right size if needed, then perform the forward - // pass. - networkOutput.set_size(network.OutputSize(), batchSize); - - network.Forward(predictors.cols(begin, begin + batchSize - 1), networkOutput); - - const typename MatType::elem_type obj = outputLayer.Forward(networkOutput, - responses.cols(begin, begin + batchSize - 1)) + network.Loss(); - - // Now perform the backward pass. - outputLayer.Backward(networkOutput, - responses.cols(begin, begin + batchSize - 1), error); - - // The delta should have the same size as the input. - networkDelta.set_size(predictors.n_rows, batchSize); - network.Backward(networkOutput, error, networkDelta); - - // Now compute the gradients. - // The gradient should have the same size as the parameters. - gradient.set_size(parameters.n_rows, parameters.n_cols); - network.Gradient(predictors.cols(begin, begin + batchSize - 1), error, - gradient); - - return obj; -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Gradient(const MatType& parameters, - const size_t begin, - MatType& gradient, - const size_t batchSize) -{ - this->EvaluateWithGradient(parameters, begin, gradient, batchSize); -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::Shuffle() -{ - math::ShuffleData(predictors, responses, predictors, responses); -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::ResetData(MatType predictors, MatType responses) +template +void FFN::ResetData( + arma::mat predictors, arma::mat responses) { + numFunctions = responses.n_cols; this->predictors = std::move(predictors); this->responses = std::move(responses); + this->deterministic = false; + ResetDeterministic(); - // Set the network to training mode. - SetNetworkMode(true); + if (!reset) + ResetParameters(); } -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::InitializeWeights() -{ - // Set the network to testing mode. - SetNetworkMode(false); - - // Reset the network parameters with the given initialization rule. - NetworkInitialization networkInit(initializeRule); - networkInit.Initialize(network.Network(), parameters); -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::SetLayerMemory() -{ - size_t totalWeightSize = network.WeightSize(); - - Log::Assert(totalWeightSize == parameters.n_elem, - "FFN::SetLayerMemory(): total layer weight size does not match parameter " - "size!"); - - network.SetWeights(parameters.memptr()); - layerMemoryIsSet = true; -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::CheckNetwork(const std::string& functionName, - const size_t inputDimensionality, - const bool setMode, - const bool training) -{ - // If the network is empty, we can't do anything. - if (network.Network().size() == 0) - { - throw std::invalid_argument(functionName + ": cannot use network with no " - "layers!"); - } - - // Next, check that the input dimensions for each layer are correct. Note - // that this will throw an exception if the user has passed data that does not - // match this->inputDimensions. - if (!inputDimensionsAreSet) - UpdateDimensions(functionName, inputDimensionality); - - // We may need to initialize the `parameters` matrix if it is empty or the - // wrong size. - if (parameters.is_empty()) - { - InitializeWeights(); - } - else if (parameters.n_elem != network.WeightSize()) - { - parameters.clear(); - InitializeWeights(); - } - - // Make sure each layer is pointing at the right memory. - if (!layerMemoryIsSet) - SetLayerMemory(); - - // Finally, set the layers of the network to the right mode if the user - // requested it. - if (setMode) - SetNetworkMode(training); -} - -template -void FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::UpdateDimensions(const std::string& functionName, - const size_t inputDimensionality) -{ - // If the input dimensions are completely unset, then assume our input is - // flat. - if (inputDimensions.size() == 0) - inputDimensions = { inputDimensionality }; - - size_t totalInputSize = 1; - for (size_t i = 0; i < inputDimensions.size(); ++i) - totalInputSize *= inputDimensions[i]; - - if (totalInputSize != inputDimensionality && inputDimensionality != 0) - { - throw std::logic_error(functionName + ": input size does not match expected" - " size set with InputDimensions()!"); - } - - // If the input dimensions have not changed from what has been computed - // before, we can terminate early---the network already has its dimensions - // set. - if (inputDimensions == network.InputDimensions()) - { - inputDimensionsAreSet = true; - return; - } - - network.InputDimensions() = inputDimensions; - network.ComputeOutputDimensions(); - inputDimensionsAreSet = true; -} - -template +template template typename std::enable_if< - ens::traits::HasMaxIterationsSignature::value, void ->::type -FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const + HasMaxIterations + ::value, void>::type +FFN:: +WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const { if (optimizer.MaxIterations() < samples && optimizer.MaxIterations() != 0) { - Log::Warn << "The optimizer's maximum number of iterations is less than the" - << " size of the dataset; the optimizer will not pass over the entire " - << "dataset. To fix this, modify the maximum number of iterations to be" - << " at least equal to the number of points of your dataset (" - << samples << ")." << std::endl; + Log::Warn << "The optimizer's maximum number of iterations " + << "is less than the size of the dataset; the " + << "optimizer will not pass over the entire " + << "dataset. To fix this, modify the maximum " + << "number of iterations to be at least equal " + << "to the number of points of your dataset " + << "(" << samples << ")." << std::endl; } } -template +template template typename std::enable_if< - !ens::traits::HasMaxIterationsSignature::value, void ->::type -FFN< - OutputLayerType, - InitializationRuleType, - MatType ->::WarnMessageMaxIterations(OptimizerType& /* optimizer */, - size_t /* samples */) const + !HasMaxIterations + ::value, void>::type +FFN:: +WarnMessageMaxIterations(OptimizerType& /* optimizer */, size_t /* samples */) + const { - // Nothing to do here. + return; } +template +template +double FFN::Train( + arma::mat predictors, + arma::mat responses, + OptimizerType& optimizer, + CallbackTypes&&... callbacks) +{ + CheckInputShape > >(network, + predictors.n_rows, + "FFN<>::Train()"); + + ResetData(std::move(predictors), std::move(responses)); + + WarnMessageMaxIterations(optimizer, this->predictors.n_cols); + + // Train the model. + const double out = optimizer.Optimize(*this, parameter, callbacks...); + + Log::Info << "FFN::FFN(): final objective of trained model is " << out + << "." << std::endl; + return out; +} + +template +template +double FFN::Train( + arma::mat predictors, + arma::mat responses, + CallbackTypes&&... callbacks) +{ + CheckInputShape > >(network, + predictors.n_rows, + "FFN<>::Train()"); + + ResetData(std::move(predictors), std::move(responses)); + + OptimizerType optimizer; + + WarnMessageMaxIterations(optimizer, this->predictors.n_cols); + + // Train the model. + const double out = optimizer.Optimize(*this, parameter, callbacks...); + + Log::Info << "FFN::FFN(): final objective of trained model is " << out + << "." << std::endl; + return out; +} + +template +template +void FFN::Forward( + const PredictorsType& inputs, ResponsesType& results) +{ + if (parameter.is_empty()) + ResetParameters(); + + Forward(inputs); + results = boost::apply_visitor(outputParameterVisitor, network.back()); +} + +template +template +void FFN::Forward( + const PredictorsType& inputs, + ResponsesType& results, + const size_t begin, + const size_t end) +{ + boost::apply_visitor(ForwardVisitor(inputs, + boost::apply_visitor(outputParameterVisitor, network[begin])), + network[begin]); + + for (size_t i = 1; i < end - begin + 1; ++i) + { + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[begin + i - 1]), + boost::apply_visitor(outputParameterVisitor, network[begin + i])), + network[begin + i]); + } + + results = boost::apply_visitor(outputParameterVisitor, network[end]); +} + +template +template +double FFN::Backward( + const PredictorsType& inputs, + const TargetsType& targets, + GradientsType& gradients) +{ + double res = outputLayer.Forward(boost::apply_visitor( + outputParameterVisitor, network.back()), targets); + + for (size_t i = 0; i < network.size(); ++i) + { + res += boost::apply_visitor(lossVisitor, network[i]); + } + + outputLayer.Backward(boost::apply_visitor(outputParameterVisitor, + network.back()), targets, error); + + gradients = arma::zeros(parameter.n_rows, parameter.n_cols); + + Backward(); + ResetGradients(gradients); + Gradient(inputs); + + return res; +} + +template +void FFN::Predict( + arma::mat predictors, arma::mat& results) +{ + CheckInputShape > >( + network, predictors.n_rows, "FFN<>::Predict()"); + + if (parameter.is_empty()) + ResetParameters(); + + if (!deterministic) + { + deterministic = true; + ResetDeterministic(); + } + + arma::mat resultsTemp; + Forward(arma::mat(predictors.colptr(0), predictors.n_rows, 1, false, true)); + resultsTemp = boost::apply_visitor(outputParameterVisitor, + network.back()).col(0); + + results = arma::mat(resultsTemp.n_elem, predictors.n_cols); + results.col(0) = resultsTemp.col(0); + + for (size_t i = 1; i < predictors.n_cols; ++i) + { + Forward(arma::mat(predictors.colptr(i), predictors.n_rows, 1, false, true)); + + resultsTemp = boost::apply_visitor(outputParameterVisitor, + network.back()); + results.col(i) = resultsTemp.col(0); + } +} + +template +template +double FFN::Evaluate( + const PredictorsType& predictors, const ResponsesType& responses) +{ + CheckInputShape > >( + network, predictors.n_rows, "FFN<>::Evaluate()"); + + if (parameter.is_empty()) + ResetParameters(); + + if (!deterministic) + { + deterministic = true; + ResetDeterministic(); + } + + Forward(predictors); + + double res = outputLayer.Forward(boost::apply_visitor( + outputParameterVisitor, network.back()), responses); + + for (size_t i = 0; i < network.size(); ++i) + { + res += boost::apply_visitor(lossVisitor, network[i]); + } + + return res; +} + +template +double FFN::Evaluate( + const arma::mat& parameters) +{ + double res = 0; + for (size_t i = 0; i < predictors.n_cols; ++i) + res += Evaluate(parameters, i, 1, true); + + return res; +} + +template +double FFN::Evaluate( + const arma::mat& /* parameters */, + const size_t begin, + const size_t batchSize, + const bool deterministic) +{ + if (parameter.is_empty()) + ResetParameters(); + + if (deterministic != this->deterministic) + { + this->deterministic = deterministic; + ResetDeterministic(); + } + + Forward(predictors.cols(begin, begin + batchSize - 1)); + double res = outputLayer.Forward( + boost::apply_visitor(outputParameterVisitor, network.back()), + responses.cols(begin, begin + batchSize - 1)); + + for (size_t i = 0; i < network.size(); ++i) + { + res += boost::apply_visitor(lossVisitor, network[i]); + } + + return res; +} + +template +double FFN::Evaluate( + const arma::mat& parameters, const size_t begin, const size_t batchSize) +{ + return Evaluate(parameters, begin, batchSize, true); +} + +template +template +double FFN:: +EvaluateWithGradient(const arma::mat& parameters, GradType& gradient) +{ + double res = 0; + for (size_t i = 0; i < predictors.n_cols; ++i) + res += EvaluateWithGradient(parameters, i, gradient, 1); + + return res; +} + +template +template +double FFN:: +EvaluateWithGradient(const arma::mat& /* parameters */, + const size_t begin, + GradType& gradient, + const size_t batchSize) +{ + if (gradient.is_empty()) + { + if (parameter.is_empty()) + ResetParameters(); + + gradient = arma::zeros(parameter.n_rows, parameter.n_cols); + } + else + { + gradient.zeros(); + } + + if (this->deterministic) + { + this->deterministic = false; + ResetDeterministic(); + } + + Forward(predictors.cols(begin, begin + batchSize - 1)); + double res = outputLayer.Forward( + boost::apply_visitor(outputParameterVisitor, network.back()), + responses.cols(begin, begin + batchSize - 1)); + + for (size_t i = 0; i < network.size(); ++i) + { + res += boost::apply_visitor(lossVisitor, network[i]); + } + + outputLayer.Backward( + boost::apply_visitor(outputParameterVisitor, network.back()), + responses.cols(begin, begin + batchSize - 1), + error); + + Backward(); + ResetGradients(gradient); + Gradient(predictors.cols(begin, begin + batchSize - 1)); + + return res; +} + +template +void FFN::Gradient( + const arma::mat& parameters, + const size_t begin, + arma::mat& gradient, + const size_t batchSize) +{ + this->EvaluateWithGradient(parameters, begin, gradient, batchSize); +} + +template +void FFN::Shuffle() +{ + math::ShuffleData(predictors, responses, predictors, responses); +} + +template +void FFN::ResetParameters() +{ + ResetDeterministic(); + + // Reset the network parameter with the given initialization rule. + NetworkInitialization networkInit(initializeRule); + networkInit.Initialize(network, parameter); +} + +template +void FFN::ResetDeterministic() +{ + DeterministicSetVisitor deterministicSetVisitor(deterministic); + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deterministicSetVisitor)); +} + +template +void FFN::ResetGradients(arma::mat& gradient) +{ + size_t offset = 0; + for (size_t i = 0; i < network.size(); ++i) + { + offset += boost::apply_visitor(GradientSetVisitor(gradient, offset), + network[i]); + } +} + +template +template +void FFN::Forward(const InputType& input) +{ + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network.front())), + network.front()); + + if (!reset) + { + if (boost::apply_visitor(outputWidthVisitor, network.front()) != 0) + { + width = boost::apply_visitor(outputWidthVisitor, network.front()); + } + + if (boost::apply_visitor(outputHeightVisitor, network.front()) != 0) + { + height = boost::apply_visitor(outputHeightVisitor, network.front()); + } + } + + for (size_t i = 1; i < network.size(); ++i) + { + if (!reset) + { + // Set the input width. + boost::apply_visitor(SetInputWidthVisitor(width), network[i]); + + // Set the input height. + boost::apply_visitor(SetInputHeightVisitor(height), network[i]); + } + + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[i - 1]), + boost::apply_visitor(outputParameterVisitor, network[i])), network[i]); + + if (!reset) + { + // Get the output width. + if (boost::apply_visitor(outputWidthVisitor, network[i]) != 0) + { + width = boost::apply_visitor(outputWidthVisitor, network[i]); + } + + // Get the output height. + if (boost::apply_visitor(outputHeightVisitor, network[i]) != 0) + { + height = boost::apply_visitor(outputHeightVisitor, network[i]); + } + } + } + + if (!reset) + reset = true; +} + +template +void FFN::Backward() +{ + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network.back()), error, + boost::apply_visitor(deltaVisitor, network.back())), network.back()); + + for (size_t i = 2; i < network.size(); ++i) + { + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i])), + network[network.size() - i]); + } +} + +template +template +void FFN::Gradient(const InputType& input) +{ + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, network[1])), network.front()); + + for (size_t i = 1; i < network.size() - 1; ++i) + { + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[i - 1]), + boost::apply_visitor(deltaVisitor, network[i + 1])), network[i]); + } + + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - 2]), error), + network[network.size() - 1]); +} + +template +template +void FFN::serialize( + Archive& ar, const uint32_t /* version */) +{ + ar(CEREAL_NVP(parameter)); + ar(CEREAL_NVP(width)); + ar(CEREAL_NVP(height)); + + ar(CEREAL_NVP(reset)); + + // Be sure to clear other layers before loading. + if (cereal::is_loading()) + { + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deleteVisitor)); + network.clear(); + } + + ar(CEREAL_VECTOR_VARIANT_POINTER(network)); + + // If we are loading, we need to initialize the weights. + if (cereal::is_loading()) + { + size_t offset = 0; + for (size_t i = 0; i < network.size(); ++i) + { + offset += boost::apply_visitor(WeightSetVisitor(parameter, offset), + network[i]); + + boost::apply_visitor(resetVisitor, network[i]); + } + + deterministic = true; + ResetDeterministic(); + } +} + +template +void FFN::Swap(FFN& network) +{ + std::swap(outputLayer, network.outputLayer); + std::swap(initializeRule, network.initializeRule); + std::swap(width, network.width); + std::swap(height, network.height); + std::swap(reset, network.reset); + std::swap(this->network, network.network); + std::swap(predictors, network.predictors); + std::swap(responses, network.responses); + std::swap(parameter, network.parameter); + std::swap(numFunctions, network.numFunctions); + std::swap(error, network.error); + std::swap(deterministic, network.deterministic); + std::swap(delta, network.delta); + std::swap(inputParameter, network.inputParameter); + std::swap(outputParameter, network.outputParameter); + std::swap(gradient, network.gradient); +}; + +template +FFN::FFN( + const FFN& network): + outputLayer(network.outputLayer), + initializeRule(network.initializeRule), + width(network.width), + height(network.height), + reset(network.reset), + predictors(network.predictors), + responses(network.responses), + parameter(network.parameter), + numFunctions(network.numFunctions), + error(network.error), + deterministic(network.deterministic), + delta(network.delta), + inputParameter(network.inputParameter), + outputParameter(network.outputParameter), + gradient(network.gradient) +{ + // Build new layers according to source network + for (size_t i = 0; i < network.network.size(); ++i) + { + this->network.push_back(boost::apply_visitor(copyVisitor, + network.network[i])); + boost::apply_visitor(resetVisitor, this->network.back()); + } +}; + +template +FFN::FFN( + FFN&& network): + outputLayer(std::move(network.outputLayer)), + initializeRule(std::move(network.initializeRule)), + width(network.width), + height(network.height), + reset(network.reset), + predictors(std::move(network.predictors)), + responses(std::move(network.responses)), + parameter(std::move(network.parameter)), + numFunctions(network.numFunctions), + error(std::move(network.error)), + deterministic(network.deterministic), + delta(std::move(network.delta)), + inputParameter(std::move(network.inputParameter)), + outputParameter(std::move(network.outputParameter)), + gradient(std::move(network.gradient)) +{ + this->network = std::move(network.network); +}; + +template +FFN& +FFN::operator = (FFN network) +{ + Swap(network); + return *this; +}; + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/forward_decls.hpp b/src/mlpack/methods/ann/forward_decls.hpp deleted file mode 100644 index e3bfc62d82..0000000000 --- a/src/mlpack/methods/ann/forward_decls.hpp +++ /dev/null @@ -1,29 +0,0 @@ -/** - * @file forward_decls.hpp - * @author Ryan Curtin - * - * Forward declarations of network types. This is needed for some `friend` - * functionality. - */ -#ifndef MLPACK_METHODS_ANN_FORWARD_DECLS_HPP -#define MLPACK_METHODS_ANN_FORWARD_DECLS_HPP - -namespace mlpack { -namespace ann { - -// See ffn.hpp. -template -class FFN; - -// See rnn.hpp. -template -class RNN; - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/not_adapted/gan/CMakeLists.txt b/src/mlpack/methods/ann/gan/CMakeLists.txt similarity index 100% rename from src/mlpack/methods/ann/not_adapted/gan/CMakeLists.txt rename to src/mlpack/methods/ann/gan/CMakeLists.txt diff --git a/src/mlpack/methods/ann/not_adapted/gan/gan.hpp b/src/mlpack/methods/ann/gan/gan.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/gan/gan.hpp rename to src/mlpack/methods/ann/gan/gan.hpp diff --git a/src/mlpack/methods/ann/not_adapted/gan/gan_impl.hpp b/src/mlpack/methods/ann/gan/gan_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/gan/gan_impl.hpp rename to src/mlpack/methods/ann/gan/gan_impl.hpp diff --git a/src/mlpack/methods/ann/not_adapted/gan/gan_policies.hpp b/src/mlpack/methods/ann/gan/gan_policies.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/gan/gan_policies.hpp rename to src/mlpack/methods/ann/gan/gan_policies.hpp diff --git a/src/mlpack/methods/ann/not_adapted/gan/metrics/CMakeLists.txt b/src/mlpack/methods/ann/gan/metrics/CMakeLists.txt similarity index 100% rename from src/mlpack/methods/ann/not_adapted/gan/metrics/CMakeLists.txt rename to src/mlpack/methods/ann/gan/metrics/CMakeLists.txt diff --git a/src/mlpack/methods/ann/not_adapted/gan/metrics/inception_score.hpp b/src/mlpack/methods/ann/gan/metrics/inception_score.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/gan/metrics/inception_score.hpp rename to src/mlpack/methods/ann/gan/metrics/inception_score.hpp diff --git a/src/mlpack/methods/ann/not_adapted/gan/metrics/inception_score_impl.hpp b/src/mlpack/methods/ann/gan/metrics/inception_score_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/gan/metrics/inception_score_impl.hpp rename to src/mlpack/methods/ann/gan/metrics/inception_score_impl.hpp diff --git a/src/mlpack/methods/ann/not_adapted/gan/wgan_impl.hpp b/src/mlpack/methods/ann/gan/wgan_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/gan/wgan_impl.hpp rename to src/mlpack/methods/ann/gan/wgan_impl.hpp diff --git a/src/mlpack/methods/ann/not_adapted/gan/wgangp_impl.hpp b/src/mlpack/methods/ann/gan/wgangp_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/gan/wgangp_impl.hpp rename to src/mlpack/methods/ann/gan/wgangp_impl.hpp diff --git a/src/mlpack/methods/ann/init_rules/const_init.hpp b/src/mlpack/methods/ann/init_rules/const_init.hpp index 3005da003c..ec1f24deea 100644 --- a/src/mlpack/methods/ann/init_rules/const_init.hpp +++ b/src/mlpack/methods/ann/init_rules/const_init.hpp @@ -98,13 +98,7 @@ class ConstInitialization //! Get the initialization value. double const& InitValue() const { return initVal; } //! Modify the initialization value. - double& InitValue() { return initVal; } - - template - void serialize(Archive& ar, const uint32_t /* version */) - { - ar(CEREAL_NVP(initVal)); - } + double& initValue() { return initVal; } private: //! Value to be initialized with diff --git a/src/mlpack/methods/ann/init_rules/glorot_init.hpp b/src/mlpack/methods/ann/init_rules/glorot_init.hpp index 8f1c9d51b6..f2d02bbb1c 100644 --- a/src/mlpack/methods/ann/init_rules/glorot_init.hpp +++ b/src/mlpack/methods/ann/init_rules/glorot_init.hpp @@ -104,19 +104,13 @@ class GlorotInitializationType */ template void Initialize(arma::Cube& W); - - /** - * Serialize the initialization. (Nothing to serialize for this one.) - */ - template - void serialize(Archive& /* ar */, const uint32_t /* version */) { } }; // class GlorotInitializationType -template<> +template <> template inline void GlorotInitializationType::Initialize(arma::Mat& W, - const size_t rows, - const size_t cols) + const size_t rows, + const size_t cols) { if (W.is_empty()) W.set_size(rows, cols); @@ -126,7 +120,7 @@ inline void GlorotInitializationType::Initialize(arma::Mat& W, normalInit.Initialize(W, rows, cols); } -template<> +template <> template inline void GlorotInitializationType::Initialize(arma::Mat& W) { @@ -138,7 +132,7 @@ inline void GlorotInitializationType::Initialize(arma::Mat& W) normalInit.Initialize(W); } -template<> +template <> template inline void GlorotInitializationType::Initialize(arma::Mat& W, const size_t rows, @@ -153,7 +147,7 @@ inline void GlorotInitializationType::Initialize(arma::Mat& W, randomInit.Initialize(W, rows, cols); } -template<> +template <> template inline void GlorotInitializationType::Initialize(arma::Mat& W) { diff --git a/src/mlpack/methods/ann/init_rules/he_init.hpp b/src/mlpack/methods/ann/init_rules/he_init.hpp index 2608b1740f..c82ecaec6c 100644 --- a/src/mlpack/methods/ann/init_rules/he_init.hpp +++ b/src/mlpack/methods/ann/init_rules/he_init.hpp @@ -136,12 +136,6 @@ class HeInitialization for (size_t i = 0; i < W.n_slices; ++i) Initialize(W.slice(i)); } - - template - void serialize(Archive& /* ar */, const uint32_t /* version */) - { - // Nothing to do. - } }; // class HeInitialization } // namespace ann diff --git a/src/mlpack/methods/ann/init_rules/network_init.hpp b/src/mlpack/methods/ann/init_rules/network_init.hpp index 8f51d243bc..5e811ef3a4 100644 --- a/src/mlpack/methods/ann/init_rules/network_init.hpp +++ b/src/mlpack/methods/ann/init_rules/network_init.hpp @@ -14,10 +14,14 @@ #define MLPACK_METHODS_ANN_INIT_RULES_NETWORK_INIT_HPP #include -#include +#include "../visitor/reset_visitor.hpp" +#include "../visitor/weight_size_visitor.hpp" +#include "../visitor/weight_set_visitor.hpp" #include "init_rules_traits.hpp" +#include + namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -25,7 +29,7 @@ namespace ann /** Artificial Neural Network. */ { * This class is used to initialize the network with the given initialization * rule. */ -template +template class NetworkInitialization { public: @@ -50,18 +54,16 @@ class NetworkInitialization * @param parameterOffset Offset for network paramater, default 0. */ template - void Initialize(const std::vector>*>& network, - arma::Mat& parameters, - size_t parameterOffset = 0) + void Initialize(const std::vector >& network, + arma::Mat& parameter, size_t parameterOffset = 0) { - // Determine the total number of parameters/weights of the given network. - if (parameters.is_empty()) + // Determine the number of parameter/weights of the given network. + if (parameter.is_empty()) { size_t weights = 0; for (size_t i = 0; i < network.size(); ++i) - weights += network[i]->WeightSize(); - - parameters.set_size(weights, 1); + weights += boost::apply_visitor(weightSizeVisitor, network[i]); + parameter.set_size(weights, 1); } // Initialize the network layer by layer or the complete network. @@ -71,8 +73,9 @@ class NetworkInitialization { // Initialize the layer with the specified parameter/weight // initialization rule. - const size_t weight = network[i]->WeightSize(); - arma::Mat tmp = arma::mat(parameters.memptr() + offset, + const size_t weight = boost::apply_visitor(weightSizeVisitor, + network[i]); + arma::Mat tmp = arma::mat(parameter.memptr() + offset, weight, 1, false, false); initializeRule.Initialize(tmp, tmp.n_elem, 1); @@ -82,7 +85,19 @@ class NetworkInitialization } else { - initializeRule.Initialize(parameters, parameters.n_elem, 1); + initializeRule.Initialize(parameter, parameter.n_elem, 1); + } + + // Note: We can't merge the for loop into the for loop above because + // WeightSetVisitor also sets the parameter/weights of the inner modules. + // Inner Modules are held by the parent module e.g. the concat module can + // hold various other modules. + for (size_t i = 0, offset = parameterOffset; i < network.size(); ++i) + { + offset += boost::apply_visitor(WeightSetVisitor(parameter, offset), + network[i]); + + boost::apply_visitor(resetVisitor, network[i]); } } @@ -90,6 +105,12 @@ class NetworkInitialization //! Instantiated InitializationRule object for initializing the network //! parameter. InitializationRuleType initializeRule; + + //! Locally-stored reset visitor. + ResetVisitor resetVisitor; + + //! Locally-stored weight size visitor. + WeightSizeVisitor weightSizeVisitor; }; // class NetworkInitialization } // namespace ann diff --git a/src/mlpack/methods/ann/init_rules/oivs_init.hpp b/src/mlpack/methods/ann/init_rules/oivs_init.hpp index 314eabf46f..1c34fbc80f 100644 --- a/src/mlpack/methods/ann/init_rules/oivs_init.hpp +++ b/src/mlpack/methods/ann/init_rules/oivs_init.hpp @@ -47,13 +47,15 @@ namespace ann /** Artificial Neural Network. */ { * w_i &=& \hat{w} \cdot \sqrt{a_i + 1} * @f} * - * Where f is the transfer function epsilon, k custom parameters, n the number - * of neurons in the outgoing layer and gamma a parameter that defines the - * random interval. + * Where f is the transfer function epsilon, k custom parameters, n the number of + * neurons in the outgoing layer and gamma a parameter that defines the random + * interval. * * @tparam ActivationFunction The activation function used for the oivs method. */ -template +template< + class ActivationFunction = LogisticFunction +> class OivsInitialization { public: diff --git a/src/mlpack/methods/ann/init_rules/random_init.hpp b/src/mlpack/methods/ann/init_rules/random_init.hpp index 22095d33a3..e48615f4ff 100644 --- a/src/mlpack/methods/ann/init_rules/random_init.hpp +++ b/src/mlpack/methods/ann/init_rules/random_init.hpp @@ -115,13 +115,6 @@ class RandomInitialization Initialize(W.slice(i)); } - template - void serialize(Archive& ar, const uint32_t /* version */) - { - ar(CEREAL_NVP(lowerBound)); - ar(CEREAL_NVP(upperBound)); - } - private: //! The number used as lower bound. double lowerBound; diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index e69e65e23d..c96c48799c 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -3,18 +3,67 @@ set(SOURCES add.hpp add_impl.hpp + add_merge.hpp + add_merge_impl.hpp + adaptive_max_pooling.hpp + adaptive_max_pooling_impl.hpp + adaptive_mean_pooling.hpp + adaptive_mean_pooling_impl.hpp alpha_dropout.hpp alpha_dropout_impl.hpp + atrous_convolution.hpp + atrous_convolution_impl.hpp base_layer.hpp + batch_norm.hpp + batch_norm_impl.hpp + bicubic_interpolation.hpp + bicubic_interpolation_impl.hpp + bilinear_interpolation.hpp + bilinear_interpolation_impl.hpp + channel_shuffle.hpp + channel_shuffle_impl.hpp + concat.hpp + concat_impl.hpp + concat_performance.hpp + concat_performance_impl.hpp concatenate.hpp concatenate_impl.hpp + constant.hpp + constant_impl.hpp convolution.hpp convolution_impl.hpp dropconnect.hpp dropconnect_impl.hpp dropout.hpp dropout_impl.hpp + elu.hpp + elu_impl.hpp + fast_lstm.hpp + fast_lstm_impl.hpp + flatten_t_swish.hpp + flatten_t_swish_impl.hpp + flexible_relu.hpp + flexible_relu_impl.hpp + glimpse.hpp + glimpse_impl.hpp + group_norm.hpp + group_norm_impl.hpp + gru.hpp + gru_impl.hpp + hard_tanh.hpp + hard_tanh_impl.hpp + highway.hpp + highway_impl.hpp + instance_norm.hpp + instance_norm_impl.hpp + isrlu.hpp + isrlu_impl.hpp + join.hpp + join_impl.hpp layer.hpp + layer_norm.hpp + layer_norm_impl.hpp + layer_traits.hpp layer_types.hpp leaky_relu.hpp leaky_relu_impl.hpp @@ -22,22 +71,73 @@ set(SOURCES linear_impl.hpp linear_no_bias.hpp linear_no_bias_impl.hpp - linear3d.hpp - linear3d_impl.hpp log_softmax.hpp log_softmax_impl.hpp + lookup.hpp + lookup_impl.hpp + lp_pooling.hpp + lp_pooling_impl.hpp lstm.hpp lstm_impl.hpp max_pooling.hpp max_pooling_impl.hpp - multi_layer.hpp - multi_layer_impl.hpp + mean_pooling.hpp + mean_pooling_impl.hpp + minibatch_discrimination.hpp + minibatch_discrimination_impl.hpp + multihead_attention_impl.hpp + multihead_attention.hpp + multiply_constant.hpp + multiply_constant_impl.hpp + multiply_merge.hpp + multiply_merge_impl.hpp + nearest_interpolation.hpp + nearest_interpolation_impl.hpp noisylinear.hpp noisylinear_impl.hpp - padding.hpp + parametric_relu.hpp + parametric_relu_impl.hpp + pixel_shuffle.hpp + pixel_shuffle_impl.hpp + positional_encoding.hpp + positional_encoding_impl.hpp + recurrent.hpp + recurrent_impl.hpp + recurrent_attention.hpp + recurrent_attention_impl.hpp + reinforce_normal.hpp + reinforce_normal_impl.hpp + relu6.hpp + relu6_impl.hpp + reparametrization.hpp + reparametrization_impl.hpp radial_basis_function.hpp radial_basis_function_impl.hpp - serialization.hpp + select.hpp + select_impl.hpp + sequential.hpp + sequential_impl.hpp + softmax_impl.hpp + softmax.hpp + spatial_dropout.hpp + spatial_dropout_impl.hpp + subview.hpp + transposed_convolution.hpp + transposed_convolution_impl.hpp + vr_class_reward.hpp + vr_class_reward_impl.hpp + c_relu.hpp + c_relu_impl.hpp + weight_norm.hpp + weight_norm_impl.hpp + hardshrink.hpp + hardshrink_impl.hpp + celu.hpp + celu_impl.hpp + softshrink.hpp + softshrink_impl.hpp + softmin.hpp + softmin_impl.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/ann/layer/not_adapted/adaptive_max_pooling.hpp b/src/mlpack/methods/ann/layer/adaptive_max_pooling.hpp similarity index 62% rename from src/mlpack/methods/ann/layer/not_adapted/adaptive_max_pooling.hpp rename to src/mlpack/methods/ann/layer/adaptive_max_pooling.hpp index ad9219759b..fd080c42dd 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/adaptive_max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/adaptive_max_pooling.hpp @@ -13,9 +13,7 @@ #define MLPACK_METHODS_ANN_LAYER_ADAPTIVE_MAX_POOLING_HPP #include - -#include "layer.hpp" -#include "max_pooling.hpp" +#include "layer_types.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -23,17 +21,20 @@ namespace ann /** Artificial Neural Network. */ { /** * Implementation of the AdaptiveMaxPooling layer. * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the layer's Outputs. The layer automatically - * cast inputs to this type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class AdaptiveMaxPoolingType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class AdaptiveMaxPooling { public: //! Create the AdaptiveMaxPooling object. - AdaptiveMaxPoolingType(); + AdaptiveMaxPooling(); /** * Create the AdaptiveMaxPooling object. @@ -41,16 +42,15 @@ class AdaptiveMaxPoolingType : public Layer * @param outputWidth Width of the output. * @param outputHeight Height of the output. */ - AdaptiveMaxPoolingType(const size_t outputWidth, - const size_t outputHeight); + AdaptiveMaxPooling(const size_t outputWidth, + const size_t outputHeight); /** * Create the AdaptiveMaxPooling object. * - * @param outputShape A two-value tuple indicating width and height of the - * output. + * @param outputShape A two-value tuple indicating width and height of the output. */ - AdaptiveMaxPoolingType(const std::tuple& outputShape); + AdaptiveMaxPooling(const std::tuple& outputShape); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -59,7 +59,8 @@ class AdaptiveMaxPoolingType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, using 3rd-order tensors as @@ -70,30 +71,48 @@ class AdaptiveMaxPoolingType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + const OutputDataType& OutputParameter() const + { return poolingLayer.OutputParameter(); } + + //! Modify the output parameter. + OutputDataType& OutputParameter() { return poolingLayer.OutputParameter(); } + + //! Get the delta. + const OutputDataType& Delta() const { return poolingLayer.Delta(); } + //! Modify the delta. + OutputDataType& Delta() { return poolingLayer.Delta(); } + + //! Get the input width. + size_t InputWidth() const { return poolingLayer.InputWidth(); } + //! Modify the input width. + size_t& InputWidth() { return poolingLayer.InputWidth(); } + + //! Get the input height. + size_t InputHeight() const { return poolingLayer.InputHeight(); } + //! Modify the input height. + size_t& InputHeight() { return poolingLayer.InputHeight(); } //! Get the output width. - size_t const& OutputWidth() const { return outputWidth; } + size_t OutputWidth() const { return outputWidth; } //! Modify the output width. size_t& OutputWidth() { return outputWidth; } //! Get the output height. - size_t const& OutputHeight() const { return outputHeight; } + size_t OutputHeight() const { return outputHeight; } //! Modify the output height. size_t& OutputHeight() { return outputHeight; } - //! Get the number of trainable weights. - size_t WeightSize() const { return 0; } + //! Get the input size. + size_t InputSize() const { return poolingLayer.InputSize(); } - const std::vector& OutputDimensions() const - { - std::vector result(this->inputDimensions.size(), 1); - result[0] = outputWidth; - result[1] = outputHeight; - return result; - } + //! Get the output size. + size_t OutputSize() const { return poolingLayer.OutputSize(); } //! Get the size of the weights. size_t WeightSize() const { return 0; } @@ -108,7 +127,7 @@ class AdaptiveMaxPoolingType : public Layer /** * Initialize Kernel Size and Stride for Adaptive Pooling. */ - void InitializeAdaptivePadding() + void IntializeAdaptivePadding() { poolingLayer.StrideWidth() = std::floor(poolingLayer.InputWidth() / outputWidth); @@ -131,7 +150,7 @@ class AdaptiveMaxPoolingType : public Layer } //! Locally stored MaxPooling Object. - MaxPoolingType poolingLayer; + MaxPooling poolingLayer; //! Locally-stored output width. size_t outputWidth; @@ -141,12 +160,7 @@ class AdaptiveMaxPoolingType : public Layer //! Locally-stored reset parameter used to initialize the layer once. bool reset; -}; // class AdaptiveMaxPoolingType - -// Convenience typedefs. - -// Standard Adaptive max pooling layer. -typedef AdaptiveMaxPoolingType AdaptiveMaxPooling; +}; // class AdaptiveMaxPooling } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/adaptive_max_pooling_impl.hpp b/src/mlpack/methods/ann/layer/adaptive_max_pooling_impl.hpp similarity index 55% rename from src/mlpack/methods/ann/layer/not_adapted/adaptive_max_pooling_impl.hpp rename to src/mlpack/methods/ann/layer/adaptive_max_pooling_impl.hpp index f01f62bdc9..25cf195b6c 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/adaptive_max_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/adaptive_max_pooling_impl.hpp @@ -18,61 +18,61 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -AdaptiveMaxPoolingType::AdaptiveMaxPoolingType() +template +AdaptiveMaxPooling::AdaptiveMaxPooling() { // Nothing to do here. } -template -AdaptiveMaxPoolingType::AdaptiveMaxPoolingType( +template +AdaptiveMaxPooling::AdaptiveMaxPooling( const size_t outputWidth, const size_t outputHeight) : - AdaptiveMaxPoolingType(std::tuple(outputWidth, outputHeight)) + AdaptiveMaxPooling(std::tuple(outputWidth, outputHeight)) { // Nothing to do here. } -template -AdaptiveMaxPoolingType::AdaptiveMaxPoolingType( +template +AdaptiveMaxPooling::AdaptiveMaxPooling( const std::tuple& outputShape): outputWidth(std::get<0>(outputShape)), outputHeight(std::get<1>(outputShape)), reset(false) { - poolingLayer = ann::MaxPoolingType(0, 0); + poolingLayer = ann::MaxPooling<>(0, 0); } -template -void AdaptiveMaxPoolingType::Forward( - const InputType& input, OutputType& output) +template +template +void AdaptiveMaxPooling::Forward( + const arma::Mat& input, arma::Mat& output) { if (!reset) { - InitializeAdaptivePadding(); + IntializeAdaptivePadding(); reset = true; } poolingLayer.Forward(input, output); } -template -void AdaptiveMaxPoolingType::Backward( - const InputType& input, - const OutputType& gy, - OutputType& g) +template +template +void AdaptiveMaxPooling::Backward( + const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { poolingLayer.Backward(input, gy, g); } -template +template template -void AdaptiveMaxPoolingType::serialize( +void AdaptiveMaxPooling::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(outputWidth)); ar(CEREAL_NVP(outputHeight)); ar(CEREAL_NVP(reset)); diff --git a/src/mlpack/methods/ann/layer/not_adapted/adaptive_mean_pooling.hpp b/src/mlpack/methods/ann/layer/adaptive_mean_pooling.hpp similarity index 62% rename from src/mlpack/methods/ann/layer/not_adapted/adaptive_mean_pooling.hpp rename to src/mlpack/methods/ann/layer/adaptive_mean_pooling.hpp index 93f0095572..46a434ab54 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/adaptive_mean_pooling.hpp +++ b/src/mlpack/methods/ann/layer/adaptive_mean_pooling.hpp @@ -1,4 +1,3 @@ -// Maybe /** * @file methods/ann/layer/adaptive_mean_pooling.hpp * @author Kartik Dutt @@ -15,9 +14,7 @@ #define MLPACK_METHODS_ANN_LAYER_ADAPTIVE_MEAN_POOLING_HPP #include - -#include "layer.hpp" -#include "mean_pooling.hpp" +#include "layer_types.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -25,17 +22,20 @@ namespace ann /** Artificial Neural Network. */ { /** * Implementation of the AdaptiveMeanPooling. * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the layer's Outputs. The layer automatically - * cast inputs to this type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class AdaptiveMeanPoolingType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class AdaptiveMeanPooling { public: //! Create the AdaptiveMeanPooling object. - AdaptiveMeanPoolingType(); + AdaptiveMeanPooling(); /** * Create the AdaptiveMeanPooling object. @@ -43,16 +43,15 @@ class AdaptiveMeanPoolingType : public Layer * @param outputWidth Width of the output. * @param outputHeight Height of the output. */ - AdaptiveMeanPoolingType(const size_t outputWidth, - const size_t outputHeight); + AdaptiveMeanPooling(const size_t outputWidth, + const size_t outputHeight); /** * Create the AdaptiveMeanPooling object. * - * @param outputShape A two-value tuple indicating width and height of the - * output. + * @param outputShape A two-value tuple indicating width and height of the output. */ - AdaptiveMeanPoolingType(const std::tuple& outputShape); + AdaptiveMeanPooling(const std::tuple& outputShape); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -61,7 +60,8 @@ class AdaptiveMeanPoolingType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, using 3rd-order tensors as @@ -72,30 +72,48 @@ class AdaptiveMeanPoolingType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + const OutputDataType& OutputParameter() const + { return poolingLayer.OutputParameter(); } + + //! Modify the output parameter. + OutputDataType& OutputParameter() { return poolingLayer.OutputParameter(); } + + //! Get the delta. + const OutputDataType& Delta() const { return poolingLayer.Delta(); } + //! Modify the delta. + OutputDataType& Delta() { return poolingLayer.Delta(); } + + //! Get the input width. + size_t InputWidth() const { return poolingLayer.InputWidth(); } + //! Modify the input width. + size_t& InputWidth() { return poolingLayer.InputWidth(); } + + //! Get the input height. + size_t InputHeight() const { return poolingLayer.InputHeight(); } + //! Modify the input height. + size_t& InputHeight() { return poolingLayer.InputHeight(); } //! Get the output width. - size_t const& OutputWidth() const { return outputWidth; } + size_t OutputWidth() const { return outputWidth; } //! Modify the output width. size_t& OutputWidth() { return outputWidth; } //! Get the output height. - size_t const& OutputHeight() const { return outputHeight; } + size_t OutputHeight() const { return outputHeight; } //! Modify the output height. size_t& OutputHeight() { return outputHeight; } - //! Get the number of trainable weights. - size_t WeightSize() const { return 0; } + //! Get the input size. + size_t InputSize() const { return poolingLayer.InputSize(); } - const std::vector& OutputDimensions() const - { - std::vector result(this->inputDimensions); - result[0] = outputWidth; - result[1] = outputHeight; - return result; - } + //! Get the output size. + size_t OutputSize() const { return poolingLayer.OutputSize(); } //! Get the size of the weights. size_t WeightSize() const { return 0; } @@ -110,7 +128,7 @@ class AdaptiveMeanPoolingType : public Layer /** * Initialize Kernel Size and Stride for Adaptive Pooling. */ - void InitializeAdaptivePadding() + void IntializeAdaptivePadding() { poolingLayer.StrideWidth() = std::floor(poolingLayer.InputWidth() / outputWidth); @@ -133,7 +151,7 @@ class AdaptiveMeanPoolingType : public Layer } //! Locally stored MeanPooling Object. - MeanPoolingType poolingLayer; + MeanPooling poolingLayer; //! Locally-stored output width. size_t outputWidth; @@ -143,12 +161,7 @@ class AdaptiveMeanPoolingType : public Layer //! Locally-stored reset parameter used to initialize the layer once. bool reset; -}; // class AdaptiveMeanPoolingType - -// Convenience typedefs. - -// Standard Adaptive mean pooling layer. -typedef AdaptiveMeanPoolingType AdaptiveMeanPooling; +}; // class AdaptiveMeanPooling } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/adaptive_mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/adaptive_mean_pooling_impl.hpp similarity index 55% rename from src/mlpack/methods/ann/layer/not_adapted/adaptive_mean_pooling_impl.hpp rename to src/mlpack/methods/ann/layer/adaptive_mean_pooling_impl.hpp index 34a5a5b27f..c930246a7f 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/adaptive_mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/adaptive_mean_pooling_impl.hpp @@ -18,61 +18,61 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -AdaptiveMeanPoolingType::AdaptiveMeanPoolingType() +template +AdaptiveMeanPooling::AdaptiveMeanPooling() { // Nothing to do here. } -template -AdaptiveMeanPoolingType::AdaptiveMeanPoolingType( +template +AdaptiveMeanPooling::AdaptiveMeanPooling( const size_t outputWidth, const size_t outputHeight) : - AdaptiveMeanPoolingType(std::tuple(outputWidth, outputHeight)) + AdaptiveMeanPooling(std::tuple(outputWidth, outputHeight)) { // Nothing to do here. } -template -AdaptiveMeanPoolingType::AdaptiveMeanPoolingType( +template +AdaptiveMeanPooling::AdaptiveMeanPooling( const std::tuple& outputShape): outputWidth(std::get<0>(outputShape)), outputHeight(std::get<1>(outputShape)), reset(false) { - poolingLayer = ann::MeanPoolingType(0, 0); + poolingLayer = ann::MeanPooling<>(0, 0); } -template -void AdaptiveMeanPoolingType::Forward( - const InputType& input, OutputType& output) +template +template +void AdaptiveMeanPooling::Forward( + const arma::Mat& input, arma::Mat& output) { if (!reset) { - InitializeAdaptivePadding(); + IntializeAdaptivePadding(); reset = true; } poolingLayer.Forward(input, output); } -template -void AdaptiveMeanPoolingType::Backward( - const InputType& input, - const OutputType& gy, - OutputType& g) +template +template +void AdaptiveMeanPooling::Backward( + const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { poolingLayer.Backward(input, gy, g); } -template +template template -void AdaptiveMeanPoolingType::serialize( +void AdaptiveMeanPooling::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(outputWidth)); ar(CEREAL_NVP(outputHeight)); ar(CEREAL_NVP(reset)); diff --git a/src/mlpack/methods/ann/layer/add.hpp b/src/mlpack/methods/ann/layer/add.hpp index 8645f38612..5fd498e704 100644 --- a/src/mlpack/methods/ann/layer/add.hpp +++ b/src/mlpack/methods/ann/layer/add.hpp @@ -13,90 +13,98 @@ #define MLPACK_METHODS_ANN_LAYER_ADD_HPP #include -#include "layer.hpp" +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Implementation of the Add layer. The Add module applies a bias term to the - * incoming data. + * Implementation of the Add module class. The Add module applies a bias term + * to the incoming data. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class AddType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class Add { public: /** - * Create the AddType object. The output size of the layer will be the same - * as the input size. + * Create the Add object using the specified number of output units. + * + * @param outSize The number of output units. */ - AddType(); - - //! Clone the AddType object. This handles polymorphism correctly. - AddType* Clone() const { return new AddType(*this); } - - // Virtual destructor. - virtual ~AddType() { } - - //! Copy the given AddType layer. - AddType(const AddType& other); - //! Take ownership of the given AddType layer. - AddType(AddType&& other); - //! Copy the given AddType layer. - AddType& operator=(const AddType& other); - //! Take ownership of the given AddType layer. - AddType& operator=(AddType&& other); + Add(const size_t outSize = 0); /** - * Forward pass: add the bias to the input. + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. * * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** - * Backward pass: send weights backwards (the bias does not affect anything). + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards trough f. Using the results from the feed + * forward pass. * * @param * (input) The propagated input activation. * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, - const MatType& gy, - MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** - * Calculate the gradient using the output and the input activation. + * Calculate the gradient using the output delta and the input activation. * * @param * (input) The propagated input. * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const MatType& /* input */, - const MatType& error, - MatType& gradient); + template + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); - //! Return the weights of the network. - const MatType& Parameters() const { return weights; } - //! Modify the weights of the network. - MatType& Parameters() { return weights; } + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } //! Get the size of weights. size_t WeightSize() const { return outSize; } - //! Compute the output dimensions of the layer, based on the internal values - //! of `InputDimensions()`. - void ComputeOutputDimensions(); - - //! Set the weights of the layer to use the given memory. - void SetWeights(typename MatType::elem_type* weightPtr); - /** - * Serialize the layer. + * Serialize the layer */ template void serialize(Archive& ar, const uint32_t /* version */); @@ -106,11 +114,17 @@ class AddType : public Layer size_t outSize; //! Locally-stored weight object. - MatType weights; -}; // class Add + OutputDataType weights; -// Standard Add layer. -typedef AddType Add; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class Add } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/add_impl.hpp b/src/mlpack/methods/ann/layer/add_impl.hpp index 8cd582c5ab..a268956fe8 100644 --- a/src/mlpack/methods/ann/layer/add_impl.hpp +++ b/src/mlpack/methods/ann/layer/add_impl.hpp @@ -19,103 +19,51 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -AddType::AddType() : outSize(0) +template +Add::Add(const size_t outSize) : + outSize(outSize) { - // Nothing to do. + weights.set_size(WeightSize(), 1); } -template -AddType::AddType(const AddType& other) : - Layer(other), - outSize(other.outSize) +template +template +void Add::Forward( + const arma::Mat& input, arma::Mat& output) { - // Nothing to do. + output = input; + output.each_col() += weights; } -template -AddType::AddType(AddType&& other) : - Layer(std::move(other)), - outSize(std::move(other.outSize)) -{ - // Nothing to do. -} - -template -AddType& -AddType::operator=(const AddType& other) -{ - if (&other != this) - { - Layer::operator=(other); - outSize = other.outSize; - } - - return *this; -} - -template -AddType& -AddType::operator=(AddType&& other) -{ - if (&other != this) - { - Layer::operator=(std::move(other)); - outSize = std::move(other.outSize); - } - - return *this; -} - -template -void AddType::Forward(const MatType& input, MatType& output) -{ - output = input + arma::repmat(arma::vectorise(weights), 1, input.n_cols); -} - -template -void AddType::Backward( - const MatType& /* input */, - const MatType& gy, - MatType& g) +template +template +void Add::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { g = gy; } -template -void AddType::Gradient( - const MatType& /* input */, - const MatType& error, - MatType& gradient) +template +template +void Add::Gradient( + const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient) { gradient = error; } -template -void AddType::SetWeights(typename MatType::elem_type* weightPtr) -{ - // Set the weights to wrap the given memory. - MakeAlias(weights, weightPtr, 1, outSize); -} - -template -void AddType::ComputeOutputDimensions() -{ - this->outputDimensions = this->inputDimensions; - - outSize = this->outputDimensions[0]; - for (size_t i = 1; i < this->outputDimensions.size(); ++i) - outSize *= this->outputDimensions[i]; -} - -template +template template -void AddType::serialize(Archive& ar, const uint32_t /* version */) +void Add::serialize( + Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(outSize)); - ar(CEREAL_NVP(weights)); + + if (cereal::is_loading()) + weights.set_size(outSize, 1); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/add_merge.hpp b/src/mlpack/methods/ann/layer/add_merge.hpp similarity index 52% rename from src/mlpack/methods/ann/layer/not_adapted/add_merge.hpp rename to src/mlpack/methods/ann/layer/add_merge.hpp index bea6da12b5..1c0e02cebf 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/add_merge.hpp +++ b/src/mlpack/methods/ann/layer/add_merge.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/add_merge.hpp * @author Marcus Edel @@ -16,6 +15,10 @@ #include +#include "../visitor/delete_visitor.hpp" +#include "../visitor/delta_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" + #include "layer_types.hpp" namespace mlpack { @@ -25,17 +28,18 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the AddMerge module class. The AddMerge class accumulates * the output of various modules. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam CustomLayers Additional custom layers that can be added. */ template< - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename... CustomLayers > -class AddMerge : public MultiLayer +class AddMerge { public: /** @@ -44,7 +48,7 @@ class AddMerge : public MultiLayer * @param model Expose all the network modules. * @param run Call the Forward/Backward method before the output is merged. */ - AddMerge(const bool run = true); + AddMerge(const bool model = false, const bool run = true); /** * Create the AddMerge object using the specified parameters. @@ -53,7 +57,7 @@ class AddMerge : public MultiLayer * @param run Call the Forward/Backward method before the output is merged. * @param ownsLayers Delete the layers when this is deallocated. */ - AddMerge(const bool run, const bool ownsLayers); + AddMerge(const bool model, const bool run, const bool ownsLayers); //! Destructor to release allocated memory. ~AddMerge(); @@ -65,7 +69,8 @@ class AddMerge : public MultiLayer * @param * (input) Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const InputType& /* input */, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -76,9 +81,10 @@ class AddMerge : public MultiLayer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * This is the overload of Backward() that runs only a specific layer with @@ -89,9 +95,10 @@ class AddMerge : public MultiLayer * @param g The calculated gradient. * @param index The index of the layer to run. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g, + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g, const size_t index); /* @@ -101,9 +108,10 @@ class AddMerge : public MultiLayer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); /* * This is the overload of Gradient() that runs a specific layer with the @@ -114,24 +122,63 @@ class AddMerge : public MultiLayer * @param gradient The calculated gradient. * @param The index of the layer to run. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient, + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient, const size_t index); + /* + * Add a new module to the model. + * + * @param args The layer parameter. + */ + template + void Add(Args... args) { network.push_back(new LayerType(args...)); } + + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Return the model modules. + std::vector >& Model() + { + if (model) + { + return network; + } + + return empty; + } + + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + //! Get the value of run parameter. bool Run() const { return run; } //! Modify the value of run parameter. bool& Run() { return run; } - const std::vector& OutputDimensions() const - { - // Propagate input size to child layers. - for (size_t i = 0; i < this->network.size(); ++i) - this->network[i]->InputDimensions() = this->inputDimensions; - return this->network.back()->OutputDimensions(); - } - /** * Serialize the layer. */ @@ -139,6 +186,9 @@ class AddMerge : public MultiLayer void serialize(Archive& ar, const uint32_t /* version */); private: + //! Parameter which indicates if the modules should be exposed. + bool model; + //! Parameter which indicates if the Forward/Backward method should be called //! before merging the output. bool run; @@ -146,6 +196,36 @@ class AddMerge : public MultiLayer //! We need this to know whether we should delete the internally-held layers //! in the destructor. bool ownsLayers; + + //! Locally-stored network modules. + std::vector > network; + + //! Locally-stored empty list of modules. + std::vector > empty; + + //! Locally-stored delete visitor module object. + DeleteVisitor deleteVisitor; + + //! Locally-stored output parameter visitor module object. + OutputParameterVisitor outputParameterVisitor; + + //! Locally-stored delta visitor module object. + DeltaVisitor deltaVisitor; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored weight object. + OutputDataType weights; }; // class AddMerge } // namespace ann diff --git a/src/mlpack/methods/ann/layer/add_merge_impl.hpp b/src/mlpack/methods/ann/layer/add_merge_impl.hpp new file mode 100644 index 0000000000..cd891088a4 --- /dev/null +++ b/src/mlpack/methods/ann/layer/add_merge_impl.hpp @@ -0,0 +1,168 @@ +/** + * @file methods/ann/layer/add_merge_impl.hpp + * @author Marcus Edel + * + * Definition of the AddMerge module which accumulates the output of the given + * modules. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP + +// In case it hasn't yet been included. +#include "add_merge.hpp" + +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +AddMerge::AddMerge( + const bool model, const bool run) : + model(model), run(run), ownsLayers(!model) +{ + // Nothing to do here. +} + +template +AddMerge::AddMerge( + const bool model, const bool run, const bool ownsLayers) : + model(model), run(run), ownsLayers(ownsLayers) +{ + // Nothing to do here. +} + +template +AddMerge::~AddMerge() +{ + if (!model && ownsLayers) + { + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deleteVisitor)); + } +} + +template +template +void AddMerge::Forward( + const InputType& input, OutputType& output) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network[i])), + network[i]); + } + } + + output = boost::apply_visitor(outputParameterVisitor, network.front()); + for (size_t i = 1; i < network.size(); ++i) + { + output += boost::apply_visitor(outputParameterVisitor, network[i]); + } +} + +template +template +void AddMerge::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[i]), gy, + boost::apply_visitor(deltaVisitor, network[i])), network[i]); + } + + g = boost::apply_visitor(deltaVisitor, network[0]); + for (size_t i = 1; i < network.size(); ++i) + { + g += boost::apply_visitor(deltaVisitor, network[i]); + } + } + else + g = gy; +} + +template +template +void AddMerge::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g, + const size_t index) +{ + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[index]), gy, + boost::apply_visitor(deltaVisitor, network[index])), network[index]); + g = boost::apply_visitor(deltaVisitor, network[index]); +} + +template +template +void AddMerge::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */ ) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(GradientVisitor(input, error), network[i]); + } + } +} + +template +template +void AddMerge::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */, + const size_t index) +{ + boost::apply_visitor(GradientVisitor(input, error), network[index]); +} + +template +template +void AddMerge::serialize( + Archive& ar, const uint32_t /* version */) +{ + // Be sure to clear other layers before loading. + if (cereal::is_loading()) + network.clear(); + + ar(CEREAL_VECTOR_VARIANT_POINTER(network)); + ar(CEREAL_NVP(model)); + ar(CEREAL_NVP(run)); + ar(CEREAL_NVP(ownsLayers)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/alpha_dropout.hpp b/src/mlpack/methods/ann/layer/alpha_dropout.hpp index b1e63870f6..abf75d87b8 100644 --- a/src/mlpack/methods/ann/layer/alpha_dropout.hpp +++ b/src/mlpack/methods/ann/layer/alpha_dropout.hpp @@ -17,7 +17,6 @@ #define MLPACK_METHODS_ANN_LAYER_ALPHA_DROPOUT_HPP #include -#include "layer.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -41,11 +40,14 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class AlphaDropoutType : public Layer +template +class AlphaDropout { public: /** @@ -54,33 +56,17 @@ class AlphaDropoutType : public Layer * @param ratio The probability of setting a value to alphaDash. * @param alphaDash The dropout scaling parameter. */ - AlphaDropoutType(const double ratio = 0.5, - const double alphaDash = -alpha * lambda); + AlphaDropout(const double ratio = 0.5, + const double alphaDash = -alpha * lambda); /** - * Clone the AlphaDropoutType object. This handles polymorphism correctly. - */ - AlphaDropoutType* Clone() const { return new AlphaDropoutType(*this); } - - // Virtual destructor. - virtual ~AlphaDropoutType() { } - - //! Copy the given AlphaDropoutType layer. - AlphaDropoutType(const AlphaDropoutType& other); - //! Take ownership of the given AlphaDropoutType layer. - AlphaDropoutType(AlphaDropoutType&& other); - //! Copy the given AlphaDropoutType layer. - AlphaDropoutType& operator=(const AlphaDropoutType& other); - //! Take ownership of the given AlphaDropoutType layer. - AlphaDropoutType& operator=(AlphaDropoutType&& other); - - /** - * Ordinary feed forward pass of the AlphaDropout layer. + * Ordinary feed forward pass of the alpha_dropout layer. * * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the alpha_dropout layer. @@ -89,7 +75,25 @@ class AlphaDropoutType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, const MatType& gy, MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the detla. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! The value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } //! The probability of setting a value to alphaDash. double Ratio() const { return ratio; } @@ -101,10 +105,10 @@ class AlphaDropoutType : public Layer double B() const { return b; } //! Value of alphaDash. - double AlphaDash() const { return alphaDash; } + double AlphaDash() const {return alphaDash; } //! Get the mask. - const MatType& Mask() const { return mask; } + OutputDataType const& Mask() const {return mask;} //! Modify the probability of setting a value to alphaDash. As //! 'a' and 'b' depend on 'ratio', modify them as well. @@ -122,8 +126,14 @@ class AlphaDropoutType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: - //! Locally-stored mask object. - MatType mask; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored mast object. + OutputDataType mask; //! The probability of setting a value to aplhaDash. double ratio; @@ -131,6 +141,9 @@ class AlphaDropoutType : public Layer //! The low variance value of SELU activation function. double alphaDash; + //! If true dropout and scaling is disabled, see notes above. + bool deterministic; + //! Value of alpha for normalized inputs (taken from SELU). static constexpr double alpha = 1.6732632423543772848170429916717; @@ -142,9 +155,7 @@ class AlphaDropoutType : public Layer //! Value to be added to a*x for affine transformation. double b; -}; // class AlphaDropoutType - -typedef AlphaDropoutType AlphaDropout; +}; // class AlphaDropout } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp b/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp index 91a150f410..34a1b44544 100644 --- a/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp +++ b/src/mlpack/methods/ann/layer/alpha_dropout_impl.hpp @@ -22,79 +22,25 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -AlphaDropoutType::AlphaDropoutType( +template +AlphaDropout::AlphaDropout( const double ratio, const double alphaDash) : ratio(ratio), - alphaDash(alphaDash) + alphaDash(alphaDash), + deterministic(false) { Ratio(ratio); } -template -AlphaDropoutType::AlphaDropoutType(const AlphaDropoutType& other) : - Layer(other), - mask(other.mask), - ratio(other.ratio), - alphaDash(other.alphaDash), - a(other.a), - b(other.b) +template +template +void AlphaDropout::Forward( + const arma::Mat& input, arma::Mat& output) { - // Nothing to do. -} - -template -AlphaDropoutType::AlphaDropoutType(AlphaDropoutType&& other) : - Layer(std::move(other)), - mask(std::move(other.mask)), - ratio(std::move(other.ratio)), - alphaDash(std::move(other.alphaDash)), - a(std::move(other.a)), - b(std::move(other.b)) -{ - // Nothing to do. -} - -template -AlphaDropoutType& -AlphaDropoutType::operator=(const AlphaDropoutType& other) -{ - if (&other != this) - { - Layer::operator=(other); - mask = other.mask; - ratio = other.ratio; - alphaDash = other.alphaDash; - a = other.a; - b = other.b; - } - - return *this; -} - -template -AlphaDropoutType& -AlphaDropoutType::operator=(AlphaDropoutType&& other) -{ - if (&other != this) - { - Layer::operator=(std::move(other)); - mask = std::move(other.mask); - ratio = std::move(other.ratio); - alphaDash = std::move(other.alphaDash); - a = std::move(other.a); - b = std::move(other.b); - } - - return *this; -} - -template -void AlphaDropoutType::Forward(const MatType& input, MatType& output) -{ - // The dropout mask will not be multiplied during testing. - if (!this->training) + // The dropout mask will not be multiplied in the deterministic mode + // (during testing). + if (deterministic) { output = input; } @@ -103,35 +49,29 @@ void AlphaDropoutType::Forward(const MatType& input, MatType& output) // Set values to alphaDash with probability ratio. Then apply affine // transformation so as to keep mean and variance of outputs to their // original values. - mask = arma::randu(input.n_rows, input.n_cols); + mask = arma::randu< arma::Mat >(input.n_rows, input.n_cols); mask.transform( [&](double val) { return (val > ratio); } ); output = (input % mask + alphaDash * (1 - mask)) * a + b; } } -template -void AlphaDropoutType::Backward( - const MatType& /* input */, const MatType& gy, MatType& g) +template +template +void AlphaDropout::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { g = gy % mask * a; } -template +template template -void AlphaDropoutType::serialize( +void AlphaDropout::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(ratio)); ar(CEREAL_NVP(alphaDash)); ar(CEREAL_NVP(a)); ar(CEREAL_NVP(b)); - - // No need to serialize the mask, since it will be recomputed on the next - // forward pass. But we should clear it if we are loading. - if (Archive::is_loading::value) - mask.clear(); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/atrous_convolution.hpp b/src/mlpack/methods/ann/layer/atrous_convolution.hpp similarity index 82% rename from src/mlpack/methods/ann/layer/not_adapted/atrous_convolution.hpp rename to src/mlpack/methods/ann/layer/atrous_convolution.hpp index cc0bbcea9d..d6086de86f 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/atrous_convolution.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/atrous_convolution.hpp * @author Aarush Gupta @@ -47,10 +46,10 @@ template < typename ForwardConvolutionRule = NaiveConvolution, typename BackwardConvolutionRule = NaiveConvolution, typename GradientConvolutionRule = NaiveConvolution, - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class AtrousConvolution : public Layer +class AtrousConvolution { public: //! Create the AtrousConvolution object. @@ -140,7 +139,8 @@ class AtrousConvolution : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -151,9 +151,10 @@ class AtrousConvolution : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -162,90 +163,101 @@ class AtrousConvolution : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& /* input */, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } //! Get the weight of the layer. - const arma::Cube& Weight() const - { - return weight; - } + arma::cube const& Weight() const { return weight; } //! Modify the weight of the layer. - arma::Cube& Weight() { return weight; } - - const std::vector& OutputDimensions() const - { - std::vector result(inputDimensions.size(), 0); - result[0] = outputWidth; - result[1] = outputHeight; - return result; - } + arma::cube& Weight() { return weight; } //! Get the bias of the layer. - const OutputType& Bias() const { return bias; } + arma::mat const& Bias() const { return bias; } //! Modify the bias of the layer. - OutputType& Bias() { return bias; } + arma::mat& Bias() { return bias; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the input width. - const size_t& InputWidth() const { return inputWidth; } + size_t InputWidth() const { return inputWidth; } //! Modify input the width. size_t& InputWidth() { return inputWidth; } //! Get the input height. - const size_t& InputHeight() const { return inputHeight; } + size_t InputHeight() const { return inputHeight; } //! Modify the input height. size_t& InputHeight() { return inputHeight; } //! Get the output width. - const size_t& OutputWidth() const { return outputWidth; } + size_t OutputWidth() const { return outputWidth; } //! Modify the output width. size_t& OutputWidth() { return outputWidth; } //! Get the output height. - const size_t& OutputHeight() const { return outputHeight; } + size_t OutputHeight() const { return outputHeight; } //! Modify the output height. size_t& OutputHeight() { return outputHeight; } + //! Get the input size. + size_t InputSize() const { return inSize; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } + //! Get the kernel width. - const size_t& KernelWidth() const { return kernelWidth; } + size_t KernelWidth() const { return kernelWidth; } //! Modify the kernel width. size_t& KernelWidth() { return kernelWidth; } //! Get the kernel height. - const size_t& KernelHeight() const { return kernelHeight; } + size_t KernelHeight() const { return kernelHeight; } //! Modify the kernel height. size_t& KernelHeight() { return kernelHeight; } //! Get the stride width. - const size_t& StrideWidth() const { return strideWidth; } + size_t StrideWidth() const { return strideWidth; } //! Modify the stride width. size_t& StrideWidth() { return strideWidth; } //! Get the stride height. - const size_t& StrideHeight() const { return strideHeight; } + size_t StrideHeight() const { return strideHeight; } //! Modify the stride height. size_t& StrideHeight() { return strideHeight; } //! Get the dilation rate on the X axis. - const size_t& DilationWidth() const { return dilationWidth; } + size_t DilationWidth() const { return dilationWidth; } //! Modify the dilation rate on the X axis. size_t& DilationWidth() { return dilationWidth; } //! Get the dilation rate on the Y axis. - const size_t& DilationHeight() const { return dilationHeight; } + size_t DilationHeight() const { return dilationHeight; } //! Modify the dilation rate on the Y axis. size_t& DilationHeight() { return dilationHeight; } //! Get the internal Padding layer. - PaddingType const& Padding() const { return padding; } + ann::Padding<> const& Padding() const { return padding; } //! Modify the internal Padding layer. - PaddingType& Padding() { return padding; } + ann::Padding<>& Padding() { return padding; } //! Get size of the weight matrix. size_t WeightSize() const @@ -346,13 +358,13 @@ class AtrousConvolution : public Layer size_t strideHeight; //! Locally-stored weight object. - OutputType weights; + OutputDataType weights; //! Locally-stored weight object. - arma::Cube weight; + arma::cube weight; //! Locally-stored bias term object. - OutputType bias; + arma::mat bias; //! Locally-stored input width. size_t inputWidth; @@ -373,19 +385,28 @@ class AtrousConvolution : public Layer size_t dilationHeight; //! Locally-stored transformed output parameter. - arma::Cube outputTemp; + arma::cube outputTemp; //! Locally-stored transformed padded input parameter. - arma::Cube inputPaddedTemp; + arma::cube inputPaddedTemp; //! Locally-stored transformed error parameter. - arma::Cube gTemp; + arma::cube gTemp; //! Locally-stored transformed gradient parameter. - arma::Cube gradientTemp; + arma::cube gradientTemp; //! Locally-stored padding layer. - PaddingType padding; + ann::Padding<> padding; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; }; // class AtrousConvolution } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/atrous_convolution_impl.hpp b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp similarity index 81% rename from src/mlpack/methods/ann/layer/not_adapted/atrous_convolution_impl.hpp rename to src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp index d18fd1f855..2377ddee00 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/atrous_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp @@ -23,15 +23,15 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > AtrousConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::AtrousConvolution() { // Nothing to do here. @@ -41,15 +41,15 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > AtrousConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::AtrousConvolution( const size_t inSize, const size_t outSize, @@ -86,15 +86,15 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > AtrousConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::AtrousConvolution( const size_t inSize, const size_t outSize, @@ -143,49 +143,49 @@ AtrousConvolution< InitializeSamePadding(padWLeft, padWRight, padHTop, padHBottom); } - padding = PaddingType(padWLeft, padWRight, padHTop, - padHBottom); + padding = ann::Padding<>(padWLeft, padWRight, padHTop, padHBottom); } template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > void AtrousConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType ->::ResetWeights(typename OutputType::elem_type* weightsPtr) + InputDataType, + OutputDataType +>::Reset() { - weight = arma::Cube(weightsPtr, kernelWidth, - kernelHeight, outSize * inSize, false, true); - bias = OutputType(weightsPtr + weight.n_elem, outSize, 1, false, true); + weight = arma::cube(weights.memptr(), kernelWidth, kernelHeight, + outSize * inSize, false, false); + bias = arma::mat(weights.memptr() + weight.n_elem, + outSize, 1, false, false); } template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > +template void AtrousConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType ->::Forward(const InputType& input, OutputType& output) + InputDataType, + OutputDataType +>::Forward(const arma::Mat& input, arma::Mat& output) { batchSize = input.n_cols; - arma::Cube inputTemp( - const_cast(input).memptr(), inputWidth, inputHeight, inSize * - batchSize, false, false); + arma::cube inputTemp(const_cast&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); if (padding.PadWLeft() != 0 || padding.PadWRight() != 0 || padding.PadHTop() != 0 || padding.PadHBottom() != 0) @@ -207,8 +207,8 @@ void AtrousConvolution< padding.PadHTop(), padding.PadHBottom(), dilationHeight); output.set_size(wConv * hConv * outSize, batchSize); - outputTemp = arma::Cube(output.memptr(), - wConv, hConv, outSize * batchSize, false, false); + outputTemp = arma::Cube(output.memptr(), wConv, hConv, + outSize * batchSize, false, false); outputTemp.zeros(); for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < @@ -222,7 +222,7 @@ void AtrousConvolution< for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { - OutputType convOutput; + arma::Mat convOutput; if (padding.PadWLeft() != 0 || padding.PadWRight() != 0 || padding.PadHTop() != 0 || padding.PadHBottom() != 0) @@ -252,24 +252,25 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > +template void AtrousConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType ->::Backward(const InputType& /* input */, const OutputType& gy, OutputType& g) + InputDataType, + OutputDataType +>::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { - arma::Cube mappedError( - ((OutputType&) gy).memptr(), outputWidth, outputHeight, outSize * - batchSize, false, false); + arma::cube mappedError(((arma::Mat&) gy).memptr(), outputWidth, + outputHeight, outSize * batchSize, false, false); g.set_size(inputWidth * inputHeight * inSize, batchSize); - gTemp = arma::Cube(g.memptr(), inputWidth, - inputHeight, inSize * batchSize, false, false); + gTemp = arma::Cube(g.memptr(), inputWidth, inputHeight, + inSize * batchSize, false, false); gTemp.zeros(); for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < @@ -283,7 +284,7 @@ void AtrousConvolution< for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { - OutputType output, rotatedFilter; + arma::Mat output, rotatedFilter; Rotate180(weight.slice(outMapIdx), rotatedFilter); BackwardConvolutionRule::Convolution(mappedError.slice(outMap), @@ -310,30 +311,29 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > +template void AtrousConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - arma::Cube mappedError( - ((OutputType&) error).memptr(), outputWidth, outputHeight, outSize * - batchSize, false, false); - arma::Cube inputTemp( - const_cast(input).memptr(), inputWidth, inputHeight, - inSize * batchSize, false, false); + arma::cube mappedError(((arma::Mat&) error).memptr(), outputWidth, + outputHeight, outSize * batchSize, false, false); + arma::cube inputTemp(const_cast&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); gradient.set_size(weights.n_elem, 1); - gradientTemp = arma::Cube(gradient.memptr(), - weight.n_rows, weight.n_cols, weight.n_slices, false, false); + gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, + weight.n_cols, weight.n_slices, false, false); gradientTemp.zeros(); for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < @@ -347,7 +347,7 @@ void AtrousConvolution< for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { - InputType inputSlice; + arma::Mat inputSlice; if (padding.PadWLeft() != 0 || padding.PadWRight() != 0 || padding.PadHTop() != 0 || padding.PadHBottom() != 0) { @@ -358,9 +358,9 @@ void AtrousConvolution< inputSlice = inputTemp.slice(inMap + batchCount * inSize); } - OutputType deltaSlice = mappedError.slice(outMap); + arma::Mat deltaSlice = mappedError.slice(outMap); - OutputType output; + arma::Mat output; GradientConvolutionRule::Convolution(inputSlice, deltaSlice, output, strideWidth, strideHeight, 1, 1); @@ -404,20 +404,18 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > template void AtrousConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::serialize(Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(outSize)); ar(CEREAL_NVP(batchSize)); @@ -432,22 +430,27 @@ void AtrousConvolution< ar(CEREAL_NVP(dilationWidth)); ar(CEREAL_NVP(dilationHeight)); ar(CEREAL_NVP(padding)); - ar(CEREAL_NVP(weights)); + + if (cereal::is_loading()) + { + weights.set_size((outSize * inSize * kernelWidth * kernelHeight) + outSize, + 1); + } } template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > void AtrousConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::InitializeSamePadding(size_t& padWLeft, size_t& padWRight, size_t& padHTop, diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index 4762ee0b59..e2d7aaf809 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -30,7 +30,6 @@ #include #include #include -#include "layer.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -41,83 +40,102 @@ namespace ann /** Artificial Neural Network. */ { * * A few convenience typedefs are given: * - * - Sigmoid - * - ReLU - * - TanH - * - Softplus - * - HardSigmoid - * - Swish - * - Mish - * - LiSHT - * - GELU - * - ELiSH - * - Elliot - * - Gaussian - * - HardSwish - * - TanhExp - * - SILU + * - SigmoidLayer + * - IdentityLayer + * - ReLULayer + * - TanHLayer + * - SoftplusLayer + * - HardSigmoidLayer + * - SwishLayer + * - MishLayer + * - LiSHTLayer + * - GELULayer + * - ELiSHLayer + * - ElliotLayer + * - GaussianLayer + * - HardSwishLayer + * - TanhExpLayer + * - SILULayer * * @tparam ActivationFunction Activation function used for the embedding layer. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ template < class ActivationFunction = LogisticFunction, - typename MatType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class BaseLayer : public Layer +class BaseLayer { public: /** * Create the BaseLayer object. */ - BaseLayer() : Layer() + BaseLayer() { // Nothing to do here. } - // Virtual destructor. - virtual ~BaseLayer() { } - - // No copy constructor or operators needed here, since the class has no - // members. - - //! Clone the BaseLayer object. This handles polymorphism correctly. - BaseLayer* Clone() const { return new BaseLayer(*this); } - /** - * Forward pass: apply the activation to the inputs. + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. * * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output) + template + void Forward(const InputType& input, OutputType& output) { ActivationFunction::Fn(input, output); } /** - * Backward pass: compute the function f(x) by propagating x backwards through - * f, using the results from the forward pass. + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards trough f. Using the results from the feed + * forward pass. * * @param input The propagated input activation. * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& input, const MatType& gy, MatType& g) + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { - MatType derivative; + arma::Mat derivative; ActivationFunction::Deriv(input, derivative); g = gy % derivative; } + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + /** * Serialize the layer. */ template - void serialize(Archive& ar, const uint32_t /* version */) + void serialize(Archive& /* ar */, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - // Nothing to serialize. + /* Nothing to do here */ } + + private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; }; // class BaseLayer // Convenience typedefs. @@ -125,122 +143,179 @@ class BaseLayer : public Layer /** * Standard Sigmoid-Layer using the logistic activation function. */ -typedef BaseLayer Sigmoid; +template < + class ActivationFunction = LogisticFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using SigmoidLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; -template -using SigmoidType = BaseLayer; +/** + * Standard Identity-Layer using the identity activation function. + */ +template < + class ActivationFunction = IdentityFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using IdentityLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard rectified linear unit non-linearity layer. */ -typedef BaseLayer ReLU; - -template -using ReLUType = BaseLayer; +template < + class ActivationFunction = RectifierFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using ReLULayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard hyperbolic tangent layer. */ -typedef BaseLayer TanH; - -template -using TanHType = BaseLayer; +template < + class ActivationFunction = TanhFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using TanHLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard Softplus-Layer using the Softplus activation function. */ -typedef BaseLayer SoftPlus; - -template -using SoftPlusType = BaseLayer; +template < + class ActivationFunction = SoftplusFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using SoftPlusLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard HardSigmoid-Layer using the HardSigmoid activation function. */ -typedef BaseLayer HardSigmoid; - -template -using HardSigmoidType = BaseLayer; +template < + class ActivationFunction = HardSigmoidFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using HardSigmoidLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard Swish-Layer using the Swish activation function. */ -typedef BaseLayer Swish; - -template -using SwishType = BaseLayer; +template < + class ActivationFunction = SwishFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using SwishFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard Mish-Layer using the Mish activation function. */ -typedef BaseLayer Mish; - -template -using MishType = BaseLayer; +template < + class ActivationFunction = MishFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using MishFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard LiSHT-Layer using the LiSHT activation function. */ -typedef BaseLayer LiSHT; - -template -using LiSHTType = BaseLayer; +template < + class ActivationFunction = LiSHTFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using LiSHTFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard GELU-Layer using the GELU activation function. */ -typedef BaseLayer GELU; - -template -using GELUType = BaseLayer; +template < + class ActivationFunction = GELUFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using GELUFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard Elliot-Layer using the Elliot activation function. */ -typedef BaseLayer Elliot; - -template -using ElliotType = BaseLayer; +template < + class ActivationFunction = ElliotFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using ElliotFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard ELiSH-Layer using the ELiSH activation function. */ -typedef BaseLayer Elish; - -template -using ElishType = BaseLayer; +template < + class ActivationFunction = ElishFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using ElishFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard Gaussian-Layer using the Gaussian activation function. */ -typedef BaseLayer Gaussian; - -template -using GaussianType = BaseLayer; +template < + class ActivationFunction = GaussianFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using GaussianFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard HardSwish-Layer using the HardSwish activation function. */ -typedef BaseLayer HardSwish; - -template -using HardSwishType = BaseLayer; +template < + class ActivationFunction = HardSwishFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using HardSwishFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard TanhExp-Layer using the TanhExp activation function. */ -typedef BaseLayer TanhExp; - -template -using TanhExpType = BaseLayer; +template < + class ActivationFunction = TanhExpFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using TanhExpFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; /** * Standard SILU-Layer using the SILU activation function. */ -typedef BaseLayer SILU; - -template -using SILUType = BaseLayer; +template < + class ActivationFunction = SILUFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using SILUFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType +>; } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp similarity index 69% rename from src/mlpack/methods/ann/layer/not_adapted/batch_norm.hpp rename to src/mlpack/methods/ann/layer/batch_norm.hpp index 634fbe2c29..52df427dba 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -50,10 +50,10 @@ namespace ann /** Artificial Neural Network. */ { * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class BatchNorm : public Layer +class BatchNorm { public: //! Create the BatchNorm object. @@ -74,9 +74,9 @@ class BatchNorm : public Layer const double momentum = 0.1); /** - * Reset the layer parameters. + * Reset the layer parameters */ - void SetWeights(typename OutputType::elem_type* weightsPtr); + void Reset(); /** * Forward pass of the Batch Normalization layer. Transforms the input data @@ -86,7 +86,8 @@ class BatchNorm : public Layer * @param input Input data for the layer * @param output Resulting output activations. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Backward pass through the layer. @@ -95,9 +96,10 @@ class BatchNorm : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activations. @@ -106,24 +108,45 @@ class BatchNorm : public Layer * @param error The calculated error * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - const OutputType& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the value of deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of deterministic parameter. + bool& Deterministic() { return deterministic; } //! Get the mean over the training data. - const OutputType& TrainingMean() const { return runningMean; } + OutputDataType const& TrainingMean() const { return runningMean; } //! Modify the mean over the training data. - OutputType& TrainingMean() { return runningMean; } + OutputDataType& TrainingMean() { return runningMean; } //! Get the variance over the training data. - const OutputType& TrainingVariance() const { return runningVariance; } + OutputDataType const& TrainingVariance() const { return runningVariance; } //! Modify the variance over the training data. - OutputType& TrainingVariance() { return runningVariance; } + OutputDataType& TrainingVariance() { return runningVariance; } //! Get the number of input units / channels. size_t InputSize() const { return size; } @@ -164,19 +187,25 @@ class BatchNorm : public Layer bool loading; //! Locally-stored scale parameter. - OutputType gamma; + OutputDataType gamma; //! Locally-stored shift parameter. - OutputType beta; + OutputDataType beta; //! Locally-stored mean object. - OutputType mean; + OutputDataType mean; //! Locally-stored variance object. - OutputType variance; + OutputDataType variance; //! Locally-stored parameters. - OutputType weights; + OutputDataType weights; + + /** + * If true then mean and variance over the training set will be considered + * instead of being calculated over the batch. + */ + bool deterministic; //! Locally-stored running mean/variance counter. size_t count; @@ -186,16 +215,25 @@ class BatchNorm : public Layer double averageFactor; //! Locally-stored mean object. - OutputType runningMean; + OutputDataType runningMean; //! Locally-stored variance object. - OutputType runningVariance; + OutputDataType runningVariance; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; //! Locally-stored normalized input. - arma::Cube normalized; + arma::cube normalized; //! Locally-stored zero mean input. - arma::Cube inputMean; + arma::cube inputMean; }; // class BatchNorm } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp similarity index 65% rename from src/mlpack/methods/ann/layer/not_adapted/batch_norm_impl.hpp rename to src/mlpack/methods/ann/layer/batch_norm_impl.hpp index dadc719117..1b6637928c 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm_impl.hpp @@ -21,21 +21,22 @@ namespace mlpack { namespace ann { /** Artificial Neural Network. */ -template -BatchNorm::BatchNorm() : +template +BatchNorm::BatchNorm() : size(0), eps(1e-8), average(true), momentum(0.0), loading(false), + deterministic(false), count(0), averageFactor(0.0) { // Nothing to do here. } -template -BatchNorm::BatchNorm( +template +BatchNorm::BatchNorm( const size_t size, const double eps, const bool average, @@ -45,6 +46,7 @@ BatchNorm::BatchNorm( average(average), momentum(momentum), loading(false), + deterministic(false), count(0), averageFactor(0.0) { @@ -53,14 +55,13 @@ BatchNorm::BatchNorm( runningVariance.ones(size, 1); } -template -void BatchNorm::SetWeights( - typename OutputType::elem_type* weightsPtr) +template +void BatchNorm::Reset() { // Gamma acts as the scaling parameters for the normalized output. - gamma = OutputType(weightsPtr, size, 1, false, false); + gamma = arma::mat(weights.memptr(), size, 1, false, false); // Beta acts as the shifting parameters for the normalized output. - beta = OutputType(weightsPtr + gamma.n_elem, size, 1, false, false); + beta = arma::mat(weights.memptr() + gamma.n_elem, size, 1, false, false); if (!loading) { @@ -68,16 +69,18 @@ void BatchNorm::SetWeights( beta.fill(0.0); } + deterministic = false; loading = false; } -template -void BatchNorm::Forward( - const InputType& input, - OutputType& output) +template +template +void BatchNorm::Forward( + const arma::Mat& input, + arma::Mat& output) { - Log::Assert(input.n_rows % size == 0, "Input features must be divisible " - "by feature maps."); + Log::Assert(input.n_rows % size == 0, "Input features must be divisible \ + by feature maps."); const size_t batchSize = input.n_cols; const size_t inputSize = input.n_rows / size; @@ -86,7 +89,7 @@ void BatchNorm::Forward( output.set_size(arma::size(input)); // We will calculate minibatch norm on each channel / feature map. - if (this->training) + if (!deterministic) { // Check only during training, batch-size can be one during inference. if (batchSize == 1 && inputSize == 1) @@ -98,14 +101,12 @@ void BatchNorm::Forward( // Input corresponds to output from convolution layer. // Use a cube for simplicity. - arma::Cube inputTemp( - const_cast(input).memptr(), inputSize, size, batchSize, - false, false); + arma::cube inputTemp(const_cast&>(input).memptr(), + inputSize, size, batchSize, false, false); // Initialize output to same size and values for convenience. - arma::Cube outputTemp( - const_cast(output).memptr(), inputSize, size, batchSize, - false, false); + arma::cube outputTemp(const_cast&>(output).memptr(), + inputSize, size, batchSize, false, false); outputTemp = inputTemp; // Calculate mean and variance over all channels. @@ -151,9 +152,8 @@ void BatchNorm::Forward( { // Normalize the input and scale and shift the output. output = input; - arma::Cube outputTemp( - const_cast(output).memptr(), input.n_rows / size, size, - batchSize, false, false); + arma::cube outputTemp(const_cast&>(output).memptr(), + input.n_rows / size, size, batchSize, false, false); outputTemp.each_slice() -= arma::repmat(runningMean.t(), input.n_rows / size, 1); @@ -166,29 +166,28 @@ void BatchNorm::Forward( } } -template -void BatchNorm::Backward( - const InputType& input, - const OutputType& gy, - OutputType& g) +template +template +void BatchNorm::Backward( + const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); g.set_size(arma::size(input)); - arma::Cube gyTemp( - const_cast(gy).memptr(), input.n_rows / size, size, - input.n_cols, false, false); - arma::Cube gTemp( - const_cast(g).memptr(), input.n_rows / size, size, - input.n_cols, false, false); + arma::cube gyTemp(const_cast&>(gy).memptr(), + input.n_rows / size, size, input.n_cols, false, false); + arma::cube gTemp(const_cast&>(g).memptr(), + input.n_rows / size, size, input.n_cols, false, false); // Step 1: dl / dxhat. - arma::Cube norm = - gyTemp.each_slice() % arma::repmat(gamma.t(), input.n_rows / size, 1); + arma::cube norm = gyTemp.each_slice() % arma::repmat(gamma.t(), + input.n_rows / size, 1); // Step 2: sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. - OutputType temp = arma::sum(norm % inputMean, 2); - OutputType vars = temp % arma::repmat(arma::pow(stdInv, 3), + arma::mat temp = arma::sum(norm % inputMean, 2); + arma::mat vars = temp % arma::repmat(arma::pow(stdInv, 3), input.n_rows / size, 1) * -0.5; // Step 3: dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + @@ -199,25 +198,25 @@ void BatchNorm::Backward( // Step 4: sum (dl / dxhat * -1 / stdInv) + variance * // (sum -2 * (x - mu)) / m. - OutputType normTemp = arma::sum(norm.each_slice() % + arma::mat normTemp = arma::sum(norm.each_slice() % arma::repmat(-stdInv, input.n_rows / size, 1) , 2) / input.n_cols; gTemp.each_slice() += normTemp; } -template -void BatchNorm::Gradient( - const InputType& /* input */, - const OutputType& error, - OutputType& gradient) +template +template +void BatchNorm::Gradient( + const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient) { gradient.set_size(size + size, 1); - arma::Cube errorTemp( - const_cast(error).memptr(), error.n_rows / size, size, - error.n_cols, false, false); + arma::cube errorTemp(const_cast&>(error).memptr(), + error.n_rows / size, size, error.n_cols, false, false); // Step 5: dl / dy * xhat. - OutputType temp = arma::sum(arma::sum(normalized % errorTemp, 0), 2); + arma::mat temp = arma::sum(arma::sum(normalized % errorTemp, 0), 2); gradient.submat(0, 0, gamma.n_elem - 1, 0) = temp.t(); // Step 6: dl / dy. @@ -225,27 +224,22 @@ void BatchNorm::Gradient( gradient.submat(gamma.n_elem, 0, gradient.n_elem - 1, 0) = temp.t(); } -template +template template -void BatchNorm::serialize( +void BatchNorm::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(size)); + + if (cereal::is_loading()) + { + weights.set_size(size + size, 1); + loading = true; + } + ar(CEREAL_NVP(eps)); ar(CEREAL_NVP(gamma)); ar(CEREAL_NVP(beta)); - ar(CEREAL_NVP(weights)); - - if (Archive::is_loading::value) - { - // Gamma acts as the scaling parameters for the normalized output. - gamma = arma::mat(weights.memptr(), size, 1, false, false); - // Beta acts as the shifting parameters for the normalized output. - beta = arma::mat(weights.memptr() + gamma.n_elem, size, 1, false, false); - } - ar(CEREAL_NVP(count)); ar(CEREAL_NVP(averageFactor)); ar(CEREAL_NVP(momentum)); diff --git a/src/mlpack/methods/ann/layer/not_adapted/bicubic_interpolation.hpp b/src/mlpack/methods/ann/layer/bicubic_interpolation.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/bicubic_interpolation.hpp rename to src/mlpack/methods/ann/layer/bicubic_interpolation.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/bicubic_interpolation_impl.hpp b/src/mlpack/methods/ann/layer/bicubic_interpolation_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/bicubic_interpolation_impl.hpp rename to src/mlpack/methods/ann/layer/bicubic_interpolation_impl.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/bilinear_interpolation.hpp b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp similarity index 55% rename from src/mlpack/methods/ann/layer/not_adapted/bilinear_interpolation.hpp rename to src/mlpack/methods/ann/layer/bilinear_interpolation.hpp index d8dfb3a286..8595bc4d57 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/bilinear_interpolation.hpp +++ b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/bilinear_interpolation.hpp * @author Kris Singh @@ -28,30 +27,35 @@ namespace ann /** Artificial Neural Network. */ { * different known points in the grid. This way, we represent any arbitrary * point, present within the grid, as a function of those four points. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class BilinearInterpolationType : public Layer +class BilinearInterpolation { public: - //! Create the BilinearInterpolationType object. - BilinearInterpolationType(); + //! Create the Bilinear Interpolation object. + BilinearInterpolation(); /** - * The constructor for the Bilinear Interpolation. The input size will be set - * by the given input when the layer is used. + * The constructor for the Bilinear Interpolation. * + * @param inRowSize Number of input rows. + * @param inColSize Number of input columns. * @param outRowSize Number of output rows. * @param outColSize Number of output columns. + * @param depth Number of input slices. */ - BilinearInterpolationType(const size_t outRowSize, - const size_t outColSize); + BilinearInterpolation(const size_t inRowSize, + const size_t inColSize, + const size_t outRowSize, + const size_t outColSize, + const size_t depth); /** * Forward pass through the layer. The layer interpolates @@ -60,7 +64,8 @@ class BilinearInterpolationType : public Layer * @param input The input matrix. * @param output The resulting interpolated output matrix. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -73,22 +78,30 @@ class BilinearInterpolationType : public Layer * @param gradient The computed backward gradient. * @param output The resulting down-sampled output. */ - void Backward(const InputType& /*input*/, - const OutputType& gradient, - OutputType& output); + template + void Backward(const arma::Mat& /*input*/, + const arma::Mat& gradient, + arma::Mat& output); - const std::vector& OutputDimensions() const - { - std::vector result(this->inputDimensions.size(), 0); - result[0] = outRowSize; - result[1] = outColSize; - if (result.size() > 2) - { - for (size_t i = 0; i < result.size(); ++i) - result[i] = this->inputDimensions[i]; - } - return result; - } + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the row size of the input. + size_t const& InRowSize() const { return inRowSize; } + //! Modify the row size of the input. + size_t& InRowSize() { return inRowSize; } + + //! Get the column size of the input. + size_t const& InColSize() const { return inColSize; } + //! Modify the column size of the input. + size_t& InColSize() { return inColSize; } //! Get the row size of the output. size_t const& OutRowSize() const { return outRowSize; } @@ -100,6 +113,17 @@ class BilinearInterpolationType : public Layer //! Modify the column size of the output. size_t& OutColSize() { return outColSize; } + //! Get the depth of the input. + size_t const& InDepth() const { return depth; } + //! Modify the depth of the input. + size_t& InDepth() { return depth; } + + //! Get the shape of the input. + size_t InputShape() const + { + return inRowSize; + } + /** * Serialize the layer. */ @@ -107,16 +131,24 @@ class BilinearInterpolationType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally stored row size of the input. + size_t inRowSize; + //! Locally stored column size of the input. + size_t inColSize; //! Locally stored row size of the output. size_t outRowSize; - //! Locally stored column size of the input. size_t outColSize; + //! Locally stored depth of the input. + size_t depth; + //! Locally stored number of input points. + size_t batchSize; + //! Locally-stored delta object. + OutputDataType delta; + //! Locally-stored output parameter object. + OutputDataType outputParameter; }; // class BilinearInterpolation -// Standard BilinearInterpolation layer. -typedef BilinearInterpolationType BilinearInterpolation; - } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/bilinear_interpolation_impl.hpp b/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp similarity index 51% rename from src/mlpack/methods/ann/layer/not_adapted/bilinear_interpolation_impl.hpp rename to src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp index 3fd621598c..3621f099b3 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/bilinear_interpolation_impl.hpp +++ b/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp @@ -19,54 +19,69 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -BilinearInterpolationType:: -BilinearInterpolationType(): + +template +BilinearInterpolation:: +BilinearInterpolation(): + inRowSize(0), + inColSize(0), outRowSize(0), - outColSize(0) + outColSize(0), + depth(0), + batchSize(0) { // Nothing to do here. } -template -BilinearInterpolationType:: -BilinearInterpolationType(const size_t outRowSize, - const size_t outColSize) : +template +BilinearInterpolation:: +BilinearInterpolation( + const size_t inRowSize, + const size_t inColSize, + const size_t outRowSize, + const size_t outColSize, + const size_t depth): + inRowSize(inRowSize), + inColSize(inColSize), outRowSize(outRowSize), - outColSize(outColSize) + outColSize(outColSize), + depth(depth), + batchSize(0) { // Nothing to do here. } -template -void BilinearInterpolationType::Forward( - const InputType& input, OutputType& output) +template +template +void BilinearInterpolation::Forward( + const arma::Mat& input, arma::Mat& output) { - const size_t batchSize = input.n_cols; - const size_t depth = this->inputDimensions.size() <= 2 ? 1 : - std::accumulate(this->inputDimensions.begin() + 2, this->inputDimensions.end(), 0); + batchSize = input.n_cols; + if (output.is_empty()) + output.set_size(outRowSize * outColSize * depth, batchSize); + else + { + assert(output.n_rows == outRowSize * outColSize * depth); + assert(output.n_cols == batchSize); + } - assert(output.n_rows == outRowSize * outColSize * depth); - assert(output.n_cols == batchSize); + assert(inRowSize >= 2); + assert(inColSize >= 2); - assert(this->inputDimensions[0] >= 2); - assert(this->inputDimensions[1] >= 2); + arma::cube inputAsCube(const_cast&>(input).memptr(), + inRowSize, inColSize, depth * batchSize, false, false); + arma::cube outputAsCube(output.memptr(), outRowSize, outColSize, + depth * batchSize, false, true); - arma::Cube inputAsCube( - const_cast(input).memptr(), this->inputDimensions[0], - this->inputDimensions[1], depth * batchSize, false, false); - arma::Cube outputAsCube( - output.memptr(), outRowSize, outColSize, depth * batchSize, false, true); - - double scaleRow = (double) this->inputDimensions[0] / (double) outRowSize; - double scaleCol = (double) this->inputDimensions[1] / (double) outColSize; + double scaleRow = (double) inRowSize / (double) outRowSize; + double scaleCol = (double) inColSize / (double) outColSize; arma::mat22 coeffs; for (size_t i = 0; i < outRowSize; ++i) { size_t rOrigin = (size_t) std::floor(i * scaleRow); - if (rOrigin > this->inputDimensions[0] - 2) - rOrigin = this->inputDimensions[0] - 2; + if (rOrigin > inRowSize - 2) + rOrigin = inRowSize - 2; // Scaled distance of the interpolated point from the topmost row. double deltaR = i * scaleRow - rOrigin; @@ -76,8 +91,8 @@ void BilinearInterpolationType::Forward( { // Scaled distance of the interpolated point from the leftmost column. size_t cOrigin = (size_t) std::floor(j * scaleCol); - if (cOrigin > this->inputDimensions[1] - 2) - cOrigin = this->inputDimensions[1] - 2; + if (cOrigin > inColSize - 2) + cOrigin = inColSize - 2; double deltaC = j * scaleCol - cOrigin; if (deltaC > 1) @@ -96,27 +111,28 @@ void BilinearInterpolationType::Forward( } } -template -void BilinearInterpolationType::Backward( - const InputType& /*input*/, - const OutputType& gradient, - OutputType& output) +template +template +void BilinearInterpolation::Backward( + const arma::Mat& /*input*/, + const arma::Mat& gradient, + arma::Mat& output) { - const size_t batchSize = output.n_cols; - const size_t depth = this->inputDimensions.size() <= 2 ? 1 : - std::accumulate(this->inputDimensions.begin() + 2, this->inputDimensions.end(), 0); - - assert(output.n_rows == this->inputDimensions[0] * this->inputDimensions[1] * depth); + if (output.is_empty()) + output.set_size(inRowSize * inColSize * depth, batchSize); + else + { + assert(output.n_rows == inRowSize * inColSize * depth); + assert(output.n_cols == batchSize); + } assert(outRowSize >= 2); assert(outColSize >= 2); - arma::Cube gradientAsCube( - ((OutputType&) gradient).memptr(), outRowSize, outColSize, depth * - batchSize, false, false); - arma::Cube outputAsCube( - output.memptr(), this->inputDimensions[0], this->inputDimensions[1], depth * batchSize, - false, true); + arma::cube gradientAsCube(((arma::Mat&) gradient).memptr(), outRowSize, + outColSize, depth * batchSize, false, false); + arma::cube outputAsCube(output.memptr(), inRowSize, inColSize, + depth * batchSize, false, true); if (gradient.n_elem == output.n_elem) { @@ -124,17 +140,17 @@ void BilinearInterpolationType::Backward( } else { - double scaleRow = (double)(outRowSize) / this->inputDimensions[0]; - double scaleCol = (double)(outColSize) / this->inputDimensions[1]; + double scaleRow = (double)(outRowSize) / inRowSize; + double scaleCol = (double)(outColSize) / inColSize; arma::mat22 coeffs; - for (size_t i = 0; i < this->inputDimensions[0]; ++i) + for (size_t i = 0; i < inRowSize; ++i) { size_t rOrigin = (size_t) std::floor(i * scaleRow); if (rOrigin > outRowSize - 2) rOrigin = outRowSize - 2; double deltaR = i * scaleRow - rOrigin; - for (size_t j = 0; j < this->inputDimensions[1]; ++j) + for (size_t j = 0; j < inColSize; ++j) { size_t cOrigin = (size_t) std::floor(j * scaleCol); @@ -157,15 +173,16 @@ void BilinearInterpolationType::Backward( } } -template +template template -void BilinearInterpolationType::serialize( +void BilinearInterpolation::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - + ar(CEREAL_NVP(inRowSize)); + ar(CEREAL_NVP(inColSize)); ar(CEREAL_NVP(outRowSize)); ar(CEREAL_NVP(outColSize)); + ar(CEREAL_NVP(depth)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/c_relu.hpp b/src/mlpack/methods/ann/layer/c_relu.hpp similarity index 65% rename from src/mlpack/methods/ann/layer/not_adapted/c_relu.hpp rename to src/mlpack/methods/ann/layer/c_relu.hpp index e273fb43e6..365111a7d7 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/c_relu.hpp +++ b/src/mlpack/methods/ann/layer/c_relu.hpp @@ -14,12 +14,10 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { - /** + * * A concatenated ReLU has two outputs, one ReLU and one negative ReLU, * concatenated together. In other words, for positive x it produces [x, 0], * and for negative x it produces [0, x]. Because it has two outputs, @@ -40,21 +38,22 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class CReLUType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class CReLU { public: - //! Create the CReLU object. - CReLUType(); - - //! Clone the CReLUType object. This handles polymorphism correctly. - CReLUType* Clone() const { return new CReLUType(*this); } + /** + * Create the CReLU object. + */ + CReLU(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -64,6 +63,7 @@ class CReLUType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -75,17 +75,35 @@ class CReLUType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const DataType& input, const DataType& gy, DataType& g); - //! Serialize the layer. + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get size of weights. + size_t WeightSize() const { return 0; } + + /** + * Serialize the layer. + */ template void serialize(Archive& /* ar */, const uint32_t /* version */); -}; // class CReLUType -// Convenience typedefs. + private: + //! Locally-stored delta object. + OutputDataType delta; -// Standard CReLU layer. -typedef CReLUType CReLU; + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class CReLU } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/c_relu_impl.hpp b/src/mlpack/methods/ann/layer/c_relu_impl.hpp similarity index 63% rename from src/mlpack/methods/ann/layer/not_adapted/c_relu_impl.hpp rename to src/mlpack/methods/ann/layer/c_relu_impl.hpp index 9cf536b9ef..dc2dbce4eb 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/c_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/c_relu_impl.hpp @@ -18,36 +18,39 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -CReLUType::CReLUType() +template +CReLU::CReLU() { // Nothing to do here. } +template template -void CReLUType::Forward( +void CReLU::Forward( const InputType& input, OutputType& output) { output = arma::join_cols(arma::max(input, 0.0 * input), arma::max( (-1 * input), 0.0 * input)); } -template -void CReLUType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) +template +template +void CReLU::Backward( + const DataType& input, const DataType& gy, DataType& g) { - OutputType temp = gy % (input >= 0.0); + DataType temp; + temp = gy % (input >= 0.0); g = temp.rows(0, (input.n_rows / 2 - 1)) - temp.rows(input.n_rows / 2, (input.n_rows - 1)); } -template +template template -void CReLUType::serialize( - Archive& ar, +void CReLU::serialize( + Archive& /* ar */, const uint32_t /* version */) { - ar(cereal::base_class>(this)); + // Nothing to do here. } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/celu.hpp b/src/mlpack/methods/ann/layer/celu.hpp similarity index 64% rename from src/mlpack/methods/ann/layer/not_adapted/celu.hpp rename to src/mlpack/methods/ann/layer/celu.hpp index 7bc357ec75..ae508703ad 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/celu.hpp +++ b/src/mlpack/methods/ann/layer/celu.hpp @@ -25,8 +25,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -48,16 +46,18 @@ namespace ann /** Artificial Neural Network. */ { * \right. * @f} * - * When not in training mode, there is no computation of the derivative. + * In the deterministic mode, there is no computation of the derivative. * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class CELUType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class CELU { public: /** @@ -67,10 +67,7 @@ class CELUType : public Layer * * @param alpha Scale parameter for the negative factor (default = 1.0). */ - CELUType(const double alpha = 1.0); - - //! Clone the CELUType object. This handles polymorphism correctly. - CELUType* Clone() const { return new CELUType(*this); } + CELU(const double alpha = 1.0); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -79,6 +76,7 @@ class CELUType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -90,29 +88,54 @@ class CELUType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const DataType& input, const DataType& gy, DataType& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the non zero gradient. double const& Alpha() const { return alpha; } //! Modify the non zero gradient. double& Alpha() { return alpha; } - //! Serialize the layer. + //! Get the value of deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of deterministic parameter. + bool& Deterministic() { return deterministic; } + + //! Get size of weights. + size_t WeightSize() { return 0; } + + /** + * Serialize the layer. + */ template void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Locally stored first derivative of the activation function. - OutputType derivative; + arma::mat derivative; //! CELU Hyperparameter (alpha > 0). double alpha; -}; // class CELUType -// Convenience typedefs. - -// Standard CELU layer. -typedef CELUType CELU; + //! If true the derivative computation is disabled, see notes above. + bool deterministic; +}; // class CELU } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/celu_impl.hpp b/src/mlpack/methods/ann/layer/celu_impl.hpp similarity index 66% rename from src/mlpack/methods/ann/layer/not_adapted/celu_impl.hpp rename to src/mlpack/methods/ann/layer/celu_impl.hpp index 76a001b091..3cfa9b6977 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/celu_impl.hpp +++ b/src/mlpack/methods/ann/layer/celu_impl.hpp @@ -18,9 +18,10 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -CELUType::CELUType(const double alpha) : - alpha(alpha) +template +CELU::CELU(const double alpha) : + alpha(alpha), + deterministic(false) { if (alpha == 0) { @@ -29,18 +30,19 @@ CELUType::CELUType(const double alpha) : } } +template template -void CELUType::Forward( +void CELU::Forward( const InputType& input, OutputType& output) { - output = arma::ones(arma::size(input)); + output = arma::ones(arma::size(input)); for (size_t i = 0; i < input.n_elem; ++i) { output(i) = (input(i) >= 0) ? input(i) : alpha * - (std::exp(input(i) / alpha) - 1); + (std::exp(input(i) / alpha) - 1); } - if (this->training) + if (!deterministic) { derivative.set_size(arma::size(input)); for (size_t i = 0; i < input.n_elem; ++i) @@ -51,24 +53,21 @@ void CELUType::Forward( } } -template -void CELUType::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) +template +template +void CELU::Backward( + const DataType& /* input */, const DataType& gy, DataType& g) { g = gy % derivative; } -template +template template -void CELUType::serialize( +void CELU::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(alpha)); - if (Archive::is_loading::value) - derivative.clear(); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/channel_shuffle.hpp b/src/mlpack/methods/ann/layer/channel_shuffle.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/channel_shuffle.hpp rename to src/mlpack/methods/ann/layer/channel_shuffle.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/channel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/channel_shuffle_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/channel_shuffle_impl.hpp rename to src/mlpack/methods/ann/layer/channel_shuffle_impl.hpp diff --git a/src/mlpack/methods/ann/layer/concat.hpp b/src/mlpack/methods/ann/layer/concat.hpp new file mode 100644 index 0000000000..e693234f3e --- /dev/null +++ b/src/mlpack/methods/ann/layer/concat.hpp @@ -0,0 +1,263 @@ +/** + * @file methods/ann/layer/concat.hpp + * @author Marcus Edel + * @author Mehul Kumar Nirala + * + * Definition of the Concat class, which acts as a concatenation container. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_CONCAT_HPP +#define MLPACK_METHODS_ANN_LAYER_CONCAT_HPP + +#include + +#include "../visitor/delete_visitor.hpp" +#include "../visitor/delta_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" + +#include "layer_types.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Implementation of the Concat class. The Concat class works as a + * feed-forward fully connected network container which plugs various layers + * together. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam CustomLayers Additional custom layers if required. + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename... CustomLayers +> +class Concat +{ + public: + /** + * Create the Concat object using the specified parameters. + * + * @param model Expose all network modules. + * @param run Call the Forward/Backward method before the output is merged. + */ + Concat(const bool model = false, + const bool run = true); + + /** + * Create the Concat object using the specified parameters. + * + * @param inputSize A vector denoting input size of each layer added. + * @param axis Concat axis. + * @param model Expose all network modules. + * @param run Call the Forward/Backward method before the output is merged. + */ + Concat(arma::Row& inputSize, + const size_t axis, + const bool model = false, + const bool run = true); + + /** + * Destroy the layers held by the model. + */ + ~Concat(); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const arma::Mat& input, arma::Mat& output); + + /** + * Ordinary feed backward pass of a neural network, using 3rd-order tensors as + * input, calculating the function f(x) by propagating x backwards through f. + * Using the results from the feed forward pass. + * + * @param * (input) The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + /** + * This is the overload of Backward() that runs only a specific layer with + * the given input. + * + * @param * (input) The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + * @param index The index of the layer to run. + */ + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g, + const size_t index); + + /* + * Calculate the gradient using the output delta and the input activation. + * + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. + */ + template + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& /* gradient */); + + /* + * This is the overload of Gradient() that runs a specific layer with the + * given input. + * + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. + * @param The index of the layer to run. + */ + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient, + const size_t index); + + /* + * Add a new module to the model. + * + * @param args The layer parameter. + */ + template + void Add(Args... args) { network.push_back(new LayerType(args...)); } + + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + + //! Return the model modules. + std::vector >& Model() + { + if (model) + { + return network; + } + + return empty; + } + + //! Return the initial point for the optimization. + const arma::mat& Parameters() const { return weights; } + //! Modify the initial point for the optimization. + arma::mat& Parameters() { return weights; } + + //! Get the value of run parameter. + bool Run() const { return run; } + //! Modify the value of run parameter. + bool& Run() { return run; } + + arma::mat const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + arma::mat& InputParameter() { return inputParameter; } + + //! Get the output parameter. + arma::mat const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + arma::mat& OutputParameter() { return outputParameter; } + + //! Get the delta.e + arma::mat const& Delta() const { return delta; } + //! Modify the delta. + arma::mat& Delta() { return delta; } + + //! Get the gradient. + arma::mat const& Gradient() const { return gradient; } + //! Modify the gradient. + arma::mat& Gradient() { return gradient; } + + //! Get the axis of concatenation. + size_t const& ConcatAxis() const { return axis; } + + //! Get the size of the weight matrix. + size_t WeightSize() const { return 0; } + + /** + * Serialize the layer + */ + template + void serialize(Archive& ar, const uint32_t /* version */); + + private: + //! Parameter which indicates the input size of modules. + arma::Row inputSize; + + //! Parameter which indicates the axis of concatenation. + size_t axis; + + //! Parameter which indicates whether to use the axis of concatenation. + bool useAxis; + + //! Parameter which indicates if the modules should be exposed. + bool model; + + //! Parameter which indicates if the Forward/Backward method should be called + //! before merging the output. + bool run; + + //! Parameter to store channels. + size_t channels; + + //! Locally-stored network modules. + std::vector > network; + + //! Locally-stored model weights. + OutputDataType weights; + + //! Locally-stored delta visitor. + DeltaVisitor deltaVisitor; + + //! Locally-stored output parameter visitor. + OutputParameterVisitor outputParameterVisitor; + + //! Locally-stored delete visitor. + DeleteVisitor deleteVisitor; + + //! Locally-stored empty list of modules. + std::vector > empty; + + //! Locally-stored delta object. + arma::mat delta; + + //! Locally-stored input parameter object. + arma::mat inputParameter; + + //! Locally-stored output parameter object. + arma::mat outputParameter; + + //! Locally-stored gradient object. + arma::mat gradient; +}; // class Concat + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "concat_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/concat_impl.hpp b/src/mlpack/methods/ann/layer/concat_impl.hpp new file mode 100644 index 0000000000..b410361295 --- /dev/null +++ b/src/mlpack/methods/ann/layer/concat_impl.hpp @@ -0,0 +1,293 @@ +/** + * @file methods/ann/layer/concat_impl.hpp + * @author Marcus Edel + * @author Mehul Kumar Nirala + * + * Implementation of the Concat class, which acts as a concatenation contain. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_CONCAT_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_CONCAT_IMPL_HPP + +// In case it hasn't yet been included. +#include "concat.hpp" + +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +Concat::Concat( + const bool model, const bool run) : + axis(0), + useAxis(false), + model(model), + run(run), + channels(1) +{ + weights.set_size(0, 0); +} + +template +Concat::Concat( + arma::Row& inputSize, + const size_t axis, + const bool model, + const bool run) : + inputSize(inputSize), + axis(axis), + useAxis(true), + model(model), + run(run) +{ + weights.set_size(0, 0); + + // Parameters to help calculate the number of channels. + size_t oldColSize = 1, newColSize = 1; + // Axis is specified and useAxis is true. + if (useAxis) + { + // Axis is specified without input dimension. + // Throw an error. + if (inputSize.n_elem > 0) + { + // Calculate rowSize, newColSize based on the axis + // of concatenation. Finally concat along cols and + // reshape to original format i.e. (input, batch_size). + size_t i = std::min(axis + 1, (size_t) inputSize.n_elem); + for (; i < inputSize.n_elem; ++i) + { + newColSize *= inputSize[i]; + } + } + else + { + throw std::logic_error("Input dimensions not specified."); + } + } + else + { + channels = 1; + } + if (newColSize <= 0) + { + throw std::logic_error("Col size is zero."); + } + channels = newColSize / oldColSize; + inputSize.clear(); +} + +template +Concat::~Concat() +{ + if (!model) + { + // Clear memory. + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deleteVisitor)); + } +} + +template +template +void Concat::Forward( + const arma::Mat& input, arma::Mat& output) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network[i])), + network[i]); + } + } + + output = boost::apply_visitor(outputParameterVisitor, network.front()); + + // Reshape output to incorporate the channels. + output.reshape(output.n_rows / channels, output.n_cols * channels); + + for (size_t i = 1; i < network.size(); ++i) + { + arma::Mat out = boost::apply_visitor(outputParameterVisitor, + network[i]); + + out.reshape(out.n_rows / channels, out.n_cols * channels); + + // Vertically concatentate output from each layer. + output = arma::join_cols(output, out); + } + // Reshape output to its original shape. + output.reshape(output.n_rows * channels, output.n_cols / channels); +} + +template +template +void Concat::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) +{ + size_t rowCount = 0; + if (run) + { + arma::Mat delta; + arma::Mat gyTmp(((arma::Mat&) gy).memptr(), gy.n_rows / channels, + gy.n_cols * channels, false, false); + for (size_t i = 0; i < network.size(); ++i) + { + // Use rows from the error corresponding to the output from each layer. + size_t rows = boost::apply_visitor( + outputParameterVisitor, network[i]).n_rows; + + // Extract from gy the parameters for the i-th network. + delta = gyTmp.rows(rowCount / channels, (rowCount + rows) / channels - 1); + delta.reshape(delta.n_rows * channels, delta.n_cols / channels); + + boost::apply_visitor(BackwardVisitor( + boost::apply_visitor(outputParameterVisitor, + network[i]), delta, + boost::apply_visitor(deltaVisitor, network[i])), network[i]); + rowCount += rows; + } + + g = boost::apply_visitor(deltaVisitor, network[0]); + for (size_t i = 1; i < network.size(); ++i) + { + g += boost::apply_visitor(deltaVisitor, network[i]); + } + } + else + { + g = gy; + } +} + +template +template +void Concat::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g, + const size_t index) +{ + size_t rowCount = 0, rows = 0; + + for (size_t i = 0; i < index; ++i) + { + rowCount += boost::apply_visitor( + outputParameterVisitor, network[i]).n_rows; + } + rows = boost::apply_visitor(outputParameterVisitor, network[index]).n_rows; + + // Reshape gy to extract the i-th layer gy. + arma::Mat gyTmp(((arma::Mat&) gy).memptr(), gy.n_rows / channels, + gy.n_cols * channels, false, false); + + arma::Mat delta = gyTmp.rows(rowCount / channels, (rowCount + rows) / + channels - 1); + delta.reshape(delta.n_rows * channels, delta.n_cols / channels); + + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[index]), delta, + boost::apply_visitor(deltaVisitor, network[index])), network[index]); + + g = boost::apply_visitor(deltaVisitor, network[index]); +} + +template +template +void Concat::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */) +{ + if (run) + { + size_t rowCount = 0; + // Reshape error to extract the i-th layer error. + arma::Mat errorTmp(((arma::Mat&) error).memptr(), + error.n_rows / channels, error.n_cols * channels, false, false); + for (size_t i = 0; i < network.size(); ++i) + { + size_t rows = boost::apply_visitor( + outputParameterVisitor, network[i]).n_rows; + + // Extract from error the parameters for the i-th network. + arma::Mat err = errorTmp.rows(rowCount / channels, (rowCount + rows) / + channels - 1); + err.reshape(err.n_rows * channels, err.n_cols / channels); + + boost::apply_visitor(GradientVisitor(input, err), network[i]); + rowCount += rows; + } + } +} + +template +template +void Concat::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */, + const size_t index) +{ + size_t rowCount = 0; + for (size_t i = 0; i < index; ++i) + { + rowCount += boost::apply_visitor(outputParameterVisitor, + network[i]).n_rows; + } + size_t rows = boost::apply_visitor( + outputParameterVisitor, network[index]).n_rows; + + arma::Mat errorTmp(((arma::Mat&) error).memptr(), + error.n_rows / channels, error.n_cols * channels, false, false); + arma::Mat err = errorTmp.rows(rowCount / channels, (rowCount + rows) / + channels - 1); + err.reshape(err.n_rows * channels, err.n_cols / channels); + + boost::apply_visitor(GradientVisitor(input, err), network[index]); +} + +template +template +void Concat::serialize( + Archive& ar, const uint32_t /* version */) +{ + ar(CEREAL_NVP(model)); + ar(CEREAL_NVP(run)); + + // Do we have to load or save a model? + if (model) + { + // Clear memory first, if needed. + if (cereal::is_loading()) + { + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deleteVisitor)); + } + ar(CEREAL_VECTOR_VARIANT_POINTER(network)); + } +} + +} // namespace ann +} // namespace mlpack + + +#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/concat_performance.hpp b/src/mlpack/methods/ann/layer/concat_performance.hpp similarity index 67% rename from src/mlpack/methods/ann/layer/not_adapted/concat_performance.hpp rename to src/mlpack/methods/ann/layer/concat_performance.hpp index c565fa3eb0..b7ddbe1625 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/concat_performance.hpp +++ b/src/mlpack/methods/ann/layer/concat_performance.hpp @@ -24,17 +24,17 @@ namespace ann /** Artificial Neural Network. */ { * feed-forward fully connected network container which plugs performance layers * together. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < typename OutputLayerType = NegativeLogLikelihood<>, - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class ConcatPerformance : public Layer +class ConcatPerformance { public: /** @@ -43,7 +43,8 @@ class ConcatPerformance : public Layer * @param inSize The number of inputs. * @param outputLayer Output layer used to evaluate the network. */ - ConcatPerformance(OutputLayerType&& outputLayer = OutputLayerType()); + ConcatPerformance(const size_t inSize = 0, + OutputLayerType&& outputLayer = OutputLayerType()); /* * Computes the Negative log likelihood. @@ -51,7 +52,8 @@ class ConcatPerformance : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& target); + template + double Forward(const arma::Mat& input, arma::Mat& target); /** * Ordinary feed backward pass of a neural network. The negative log @@ -64,29 +66,42 @@ class ConcatPerformance : public Layer * between 1 and the number of classes. * @param output The calculated error. */ - void Backward(const InputType& input, - const OutputType& target, - OutputType& output); + template + void Backward(const arma::Mat& input, + const arma::Mat& target, + arma::Mat& output); //! Get the output parameter. - OutputType& OutputParameter() const { return outputParameter; } + OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. - OutputType& OutputParameter() { return outputParameter; } + OutputDataType& OutputParameter() { return outputParameter; } //! Get the delta. - OutputType& Delta() const { return delta; } + OutputDataType& Delta() const { return delta; } //! Modify the delta. - OutputType& Delta() { return delta; } + OutputDataType& Delta() { return delta; } + + //! Get the number of inputs. + size_t InSize() const { return inSize; } /** - * Serialize the layer. + * Serialize the layer */ template void serialize(Archive& /* ar */, const uint32_t /* version */); private: + //! Locally-stored number of inputs. + size_t inSize; + //! Instantiated outputlayer used to evaluate the network. OutputLayerType outputLayer; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; }; // class ConcatPerformance } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/concat_performance_impl.hpp b/src/mlpack/methods/ann/layer/concat_performance_impl.hpp similarity index 61% rename from src/mlpack/methods/ann/layer/not_adapted/concat_performance_impl.hpp rename to src/mlpack/methods/ann/layer/concat_performance_impl.hpp index 60def206a9..ee4d9b8f4b 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/concat_performance_impl.hpp +++ b/src/mlpack/methods/ann/layer/concat_performance_impl.hpp @@ -20,14 +20,15 @@ namespace ann /** Artificial Neural Network. */ { template< typename OutputLayerType, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > ConcatPerformance< OutputLayerType, - InputType, - OutputType ->::ConcatPerformance(OutputLayerType&& outputLayer) : + InputDataType, + OutputDataType +>::ConcatPerformance(const size_t inSize, OutputLayerType&& outputLayer) : + inSize(inSize), outputLayer(std::move(outputLayer)) { // Nothing to do here. @@ -35,51 +36,51 @@ ConcatPerformance< template< typename OutputLayerType, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > -void ConcatPerformance< +template +double ConcatPerformance< OutputLayerType, - InputType, - OutputType ->::Forward(const InputType& input, OutputType& target) + InputDataType, + OutputDataType +>::Forward(const arma::Mat& input, arma::Mat& target) { - const size_t elements = input.n_elem / inputDimensions[0]; + const size_t elements = input.n_elem / inSize; double output = 0; - for (size_t i = 0; i < input.n_elem; i += elements) + for (size_t i = 0; i < input.n_elem; i+= elements) { - InputType subInput = input.submat(i, 0, i + elements - 1, 0); + arma::mat subInput = input.submat(i, 0, i + elements - 1, 0); output += outputLayer.Forward(subInput, target); } - // TODO: what to do with output? - //return output; - return; + return output; } template< typename OutputLayerType, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > +template void ConcatPerformance< OutputLayerType, - InputType, - OutputType + InputDataType, + OutputDataType >::Backward( - const InputType& input, - const OutputType& target, - OutputType& output) + const arma::Mat& input, + const arma::Mat& target, + arma::Mat& output) { - const size_t elements = input.n_elem / inputDimensions[0]; + const size_t elements = input.n_elem / inSize; - InputType subInput = input.submat(0, 0, elements - 1, 0); - OutputType subOutput; + arma::mat subInput = input.submat(0, 0, elements - 1, 0); + arma::mat subOutput; outputLayer.Backward(subInput, target, subOutput); - output = arma::zeros(subOutput.n_elem, inputDimensions[0]); + output = arma::zeros(subOutput.n_elem, inSize); output.col(0) = subOutput; for (size_t i = elements, j = 0; i < input.n_elem; i+= elements, ++j) @@ -93,19 +94,17 @@ void ConcatPerformance< template< typename OutputLayerType, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > template void ConcatPerformance< OutputLayerType, - InputType, - OutputType + InputDataType, + OutputDataType >::serialize(Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(outputLayer)); + ar(CEREAL_NVP(inSize)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/concatenate.hpp b/src/mlpack/methods/ann/layer/concatenate.hpp index 7c21560465..561ecf2595 100644 --- a/src/mlpack/methods/ann/layer/concatenate.hpp +++ b/src/mlpack/methods/ann/layer/concatenate.hpp @@ -14,7 +14,7 @@ #define MLPACK_METHODS_ANN_LAYER_CONCATENATE_HPP #include -#include "layer.hpp" +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -22,39 +22,36 @@ namespace ann /** Artificial Neural Network. */ { /** * Implementation of the Concatenate module class. The Concatenate module * concatenates a constant given matrix to the incoming data. + * Note: Users need to use the Concat() function to provide the concat matrix. * - * The Concat() function provides the concat matrix, or it can be passed to - * the constructor. - * - * After this layer is applied, the shape of the data will be a vector. - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class ConcatenateType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class Concatenate { public: /** - * Create the ConcatenateType object using the given constant matrix as the - * data to be concatenated to the output of the forward pass. + * Create the Concatenate object using the specified number of output units. */ - ConcatenateType(const MatType& concat = MatType()); + Concatenate(); - //! Clone the ConcatenateType object. This handles polymorphism correctly. - ConcatenateType* Clone() const { return new ConcatenateType(*this); } + //! Copy constructor. + Concatenate(const Concatenate& layer); - // Virtual destructor. - virtual ~ConcatenateType() { } + //! Move constructor. + Concatenate(Concatenate&& layer); - //! Copy the given ConcatenateType layer. - ConcatenateType(const ConcatenateType& other); - //! Take ownership of the given ConcatenateType layer. - ConcatenateType(ConcatenateType&& other); - //! Copy the given ConcatenateType layer. - ConcatenateType& operator=(const ConcatenateType& other); - //! Take ownership of the given ConcatenateType layer. - ConcatenateType& operator=(ConcatenateType&& other); + //! Operator= copy constructor. + Concatenate& operator=(const Concatenate& layer); + + //! Operator= move constructor. + Concatenate& operator=(Concatenate&& layer); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -63,7 +60,8 @@ class ConcatenateType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -74,31 +72,57 @@ class ConcatenateType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, const MatType& gy, MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the concat matrix. - MatType const& Concat() const { return concat; } + OutputDataType const& Concat() const { return concat; } //! Modify the concat. - MatType& Concat() { return concat; } - - //! Compute the output dimensions of the layer based on `InputDimensions()`. - void ComputeOutputDimensions(); + OutputDataType& Concat() { return concat; } /** - * Serialize the layer. + * Serialize the layer */ template - void serialize(Archive& ar, const uint32_t /* version */); + void serialize(Archive& /* ar */, const uint32_t /* version */) + { + // Nothing to do here. + } private: - //! Matrix to be concatenated to input. - MatType concat; + //! Locally-stored number of input rows. + size_t inRows; + //! Locally-stored weight object. + OutputDataType weights; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored matrix to be concatenated to input. + OutputDataType concat; }; // class Concatenate -// Standard Concatenate layer. -typedef ConcatenateType Concatenate; - } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/concatenate_impl.hpp b/src/mlpack/methods/ann/layer/concatenate_impl.hpp index 8980e0c606..54b53471c8 100644 --- a/src/mlpack/methods/ann/layer/concatenate_impl.hpp +++ b/src/mlpack/methods/ann/layer/concatenate_impl.hpp @@ -19,106 +19,91 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -ConcatenateType:: -ConcatenateType(const MatType& concat) : - concat(concat) +template +Concatenate::Concatenate() : + inRows(0) { // Nothing to do here. } -template -ConcatenateType:: -ConcatenateType(const ConcatenateType& other) : - Layer(other), - concat(other.concat) +template +Concatenate::Concatenate( + const Concatenate& layer) : + inRows(layer.inRows), + weights(layer.weights), + delta(layer.delta), + concat(layer.concat) { - // Nothing to do. + // Nothing to to here. } -template -ConcatenateType:: -ConcatenateType(ConcatenateType&& other) : - Layer(std::move(other)), - concat(other.concat) +template +Concatenate::Concatenate(Concatenate&& layer) : + inRows(layer.inRows), + weights(std::move(layer.weights)), + delta(std::move(layer.delta)), + concat(std::move(layer.concat)) { - // Nothing to do. + // Nothing to do here. } -template -ConcatenateType& -ConcatenateType::operator=(const ConcatenateType& other) +template +Concatenate& +Concatenate:: +operator=(const Concatenate& layer) { - if (&other != this) + if (this != &layer) { - Layer::operator=(other); - concat = other.concat; + inRows = layer.inRows; + weights = layer.weights; + delta = layer.delta; + concat = layer.concat; } return *this; } -template -ConcatenateType& -ConcatenateType::operator=(ConcatenateType&& other) +template +Concatenate& +Concatenate:: +operator=(Concatenate&& layer) { - if (&other != this) + if (this != &layer) { - Layer::operator=(std::move(other)); - concat = std::move(other.concat); + inRows = layer.inRows; + weights = std::move(layer.weights); + delta = std::move(layer.delta); + concat = std::move(layer.concat); } - return *this; } -template -void ConcatenateType::Forward(const MatType& input, MatType& output) +template +template +void Concatenate::Forward( + const arma::Mat& input, arma::Mat& output) { if (concat.is_empty()) + Log::Warn << "The concat matrix has not been provided." << std::endl; + + if (input.n_cols != concat.n_cols) { - Log::Warn << "Concatenate::Forward(): the concat matrix is empty or was " - << "not provided." << std::endl; + Log::Fatal << "The number of columns of the concat matrix should be equal " + << "to the number of columns of input matrix." << std::endl; } - output.submat(0, 0, input.n_rows - 1, input.n_cols - 1) = input; - output.submat(input.n_rows, 0, output.n_rows - 1, input.n_cols - 1) = - arma::repmat(arma::vectorise(concat), 1, input.n_cols); + inRows = input.n_rows; + output = arma::join_cols(input, concat); } -template -void ConcatenateType::Backward( - const MatType& /* input */, - const MatType& gy, - MatType& g) +template +template +void Concatenate::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - // Pass back the non-concatenated part. - g = gy.submat(0, 0, gy.n_rows - 1 - concat.n_elem, gy.n_cols - 1); -} - -template -void ConcatenateType::ComputeOutputDimensions() -{ - // This flattens the input. - size_t inSize = this->inputDimensions[0]; - for (size_t i = 1; i < this->inputDimensions.size(); ++i) - inSize *= this->inputDimensions[i]; - - this->outputDimensions = std::vector(this->inputDimensions.size(), - 1); - this->outputDimensions[0] = inSize + concat.n_elem; -} - -/** - * Serialize the layer. - */ -template -template -void ConcatenateType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(concat)); + g = gy.submat(0, 0, inRows - 1, concat.n_cols - 1); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/constant.hpp b/src/mlpack/methods/ann/layer/constant.hpp similarity index 56% rename from src/mlpack/methods/ann/layer/not_adapted/constant.hpp rename to src/mlpack/methods/ann/layer/constant.hpp index bcfe997d16..9a0956ddf8 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/constant.hpp +++ b/src/mlpack/methods/ann/layer/constant.hpp @@ -15,8 +15,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -24,21 +22,18 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the constant layer. The constant layer outputs a given * constant value given any input value. * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class ConstantType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class Constant { public: - /** - * Create an empty Constant layer. - */ - ConstantType(); - /** * Create the Constant object that outputs a given constant scalar value * given any input value. @@ -46,19 +41,7 @@ class ConstantType : public Layer * @param outSize The number of output units. * @param scalar The constant value used to create the constant output. */ - ConstantType(const size_t outSize, const double scalar = 0); - - //! Copy another ConstantType. - ConstantType(const ConstantType& layer); - //! Take ownership of another ConstantType. - ConstantType(ConstantType&& layer); - //! Copy another ConstantType. - ConstantType& operator=(const ConstantType& layer); - //! Take ownership of another ConstantType. - ConstantType& operator=(ConstantType&& layer); - - //! Clone the ConstantType object. This handles polymorphism correctly. - ConstantType* Clone() const { return new ConstantType(*this); } + Constant(const size_t outSize = 0, const double scalar = 0.0); /** * Ordinary feed forward pass of a neural network. The forward pass fills the @@ -67,6 +50,7 @@ class ConstantType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -77,34 +61,52 @@ class ConstantType : public Layer * @param * (gy) The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& /* gy */, - OutputType& g); + template + void Backward(const DataType& /* input */, + const DataType& /* gy */, + DataType& g); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the output size. - const std::vector& OutputDimensions() const + size_t OutSize() const { return outSize; } + + //! Get the size of the weights. + size_t WeightSize() const { - std::vector result(this->inputDimensions.size(), 0); - result[0] = outSize; - return result; + return 0; } - //! Serialize the layer. + /** + * Serialize the layer. + */ template void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored number of input units. + size_t inSize; + //! Locally-stored number of output units. size_t outSize; //! Locally-stored constant output matrix. - OutputType constantOutput; -}; // class ConstantType + OutputDataType constantOutput; -// Convenience typedefs. + //! Locally-stored delta object. + OutputDataType delta; -// Standard HardShrink layer. -typedef ConstantType Constant; + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class ConstantLayer } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/constant_impl.hpp b/src/mlpack/methods/ann/layer/constant_impl.hpp new file mode 100644 index 0000000000..3f16311e46 --- /dev/null +++ b/src/mlpack/methods/ann/layer/constant_impl.hpp @@ -0,0 +1,65 @@ +/** + * @file methods/ann/layer/constant_impl.hpp + * @author Marcus Edel + * + * Implementation of the Constant class, which outputs a constant value given + * any input. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_CONSTANT_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_CONSTANT_IMPL_HPP + +// In case it hasn't yet been included. +#include "constant.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +Constant::Constant( + const size_t outSize, + const double scalar) : + inSize(0), + outSize(outSize) +{ + constantOutput = OutputDataType(outSize, 1); + constantOutput.fill(scalar); +} + +template +template +void Constant::Forward( + const InputType& input, OutputType& output) +{ + if (inSize == 0) + { + inSize = input.n_elem; + } + + output = constantOutput; +} + +template +template +void Constant::Backward( + const DataType& /* input */, const DataType& /* gy */, DataType& g) +{ + g = arma::zeros(inSize, 1); +} + +template +template +void Constant::serialize( + Archive& ar, const uint32_t /* version */) +{ + ar(CEREAL_NVP(constantOutput)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index d6597edadb..c325eba95d 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -20,7 +20,7 @@ #include #include -#include "layer.hpp" +#include "layer_types.hpp" #include "padding.hpp" namespace mlpack { @@ -30,7 +30,7 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the Convolution class. The Convolution class represents a * single layer of a neural network. * Example usage: - * + * * Suppose we want to pass a matrix M (2744x100) to a `Convolution` layer; * in this example, `M` was obtained from "flattening" 100 images (or Mel * cepstral coefficients, if we talk about speech, or whatever you like) of @@ -62,95 +62,100 @@ namespace ann /** Artificial Neural Network. */ { * @tparam ForwardConvolutionRule Convolution to perform forward process. * @tparam BackwardConvolutionRule Convolution to perform backward process. * @tparam GradientConvolutionRule Convolution to calculate gradient. - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ template < typename ForwardConvolutionRule = NaiveConvolution, typename BackwardConvolutionRule = NaiveConvolution, typename GradientConvolutionRule = NaiveConvolution, - typename MatType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class ConvolutionType : public Layer +class Convolution { public: - //! Create the ConvolutionType object. - ConvolutionType(); + //! Create the Convolution object. + Convolution(); /** - * Create the ConvolutionType object using the specified number of output - * maps, filter size, stride and padding parameter. + * Create the Convolution object using the specified number of input maps, + * output maps, filter size, stride and padding parameter. * - * @param maps The number of output maps. + * @param inSize The number of input maps. + * @param outSize The number of output maps. * @param kernelWidth Width of the filter/kernel. * @param kernelHeight Height of the filter/kernel. * @param strideWidth Stride of filter application in the x direction. * @param strideHeight Stride of filter application in the y direction. * @param padW Padding width of the input. * @param padH Padding height of the input. - * @param paddingType The type of padding ("valid" or "same"). Defaults to - * "none". If not specified or "none", the values for `padW` and `padH` - * will be used. + * @param inputWidth The width of the input data. + * @param inputHeight The height of the input data. + * @param paddingType The type of padding (Valid or Same). Defaults to None. */ - ConvolutionType(const size_t maps, - const size_t kernelWidth, - const size_t kernelHeight, - const size_t strideWidth = 1, - const size_t strideHeight = 1, - const size_t padW = 0, - const size_t padH = 0, - const std::string& paddingType = "none"); + Convolution(const size_t inSize, + const size_t outSize, + const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth = 1, + const size_t strideHeight = 1, + const size_t padW = 0, + const size_t padH = 0, + const size_t inputWidth = 0, + const size_t inputHeight = 0, + const std::string& paddingType = "None"); /** * Create the Convolution object using the specified number of input maps, * output maps, filter size, stride and padding parameter. * - * @param maps The number of output maps. + * @param inSize The number of input maps. + * @param outSize The number of output maps. * @param kernelWidth Width of the filter/kernel. * @param kernelHeight Height of the filter/kernel. * @param strideWidth Stride of filter application in the x direction. * @param strideHeight Stride of filter application in the y direction. - * @param padW A two-value tuple indicating padding widths of the input. The - * first value is the padding for the left side; the second value is the - * padding on the right side. - * @param padH A two-value tuple indicating padding heights of the input. The - * first value is the padding for the top; the second value is the - * padding on the bottom. - * @param paddingType The type of padding ("valid" or "same"). Defaults to - * "none". If not specified or "none", the values for `padW` and `padH` - * will be used. + * @param padW A two-value tuple indicating padding widths of the input. + * First value is padding at left side. Second value is padding on + * right side. + * @param padH A two-value tuple indicating padding heights of the input. + * First value is padding at top. Second value is padding on + * bottom. + * @param inputWidth The width of the input data. + * @param inputHeight The height of the input data. + * @param paddingType The type of padding (Valid or Same). Defaults to None. */ - ConvolutionType(const size_t maps, - const size_t kernelWidth, - const size_t kernelHeight, - const size_t strideWidth, - const size_t strideHeight, - const std::tuple& padW, - const std::tuple& padH, - const std::string& paddingType = "none"); + Convolution(const size_t inSize, + const size_t outSize, + const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth, + const size_t strideHeight, + const std::tuple& padW, + const std::tuple& padH, + const size_t inputWidth = 0, + const size_t inputHeight = 0, + const std::string& paddingType = "None"); - //! Clone the ConvolutionType object. This handles polymorphism correctly. - ConvolutionType* Clone() const { return new ConvolutionType(*this); } + //! Copy constructor. + Convolution(const Convolution& layer); - //! Copy the given ConvolutionType (but not weights). - ConvolutionType(const ConvolutionType& layer); + //! Move constructor. + Convolution(Convolution&&); - //! Take ownership of the given ConvolutionType (but not weights). - ConvolutionType(ConvolutionType&&); + //! Copy assignment operator. + Convolution& operator=(const Convolution& layer); - //! Copy the given ConvolutionType (but not weights). - ConvolutionType& operator=(const ConvolutionType& layer); - - //! Take ownership of the given ConvolutionType (but not weights). - ConvolutionType& operator=(ConvolutionType&& layer); - - // Virtual destructor. - virtual ~ConvolutionType() { } + //! Move assignment operator. + Convolution& operator=(Convolution&& layer); /* * Set the weight and bias term. */ - void SetWeights(typename MatType::elem_type* weightsPtr); + void Reset(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -159,7 +164,8 @@ class ConvolutionType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -170,91 +176,135 @@ class ConvolutionType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, - const MatType& gy, - MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); - /** + /* * Calculate the gradient using the output delta and the input activation. * * @param input The input parameter used for calculating the gradient. * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const MatType& /* input */, - const MatType& error, - MatType& gradient); + template + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - MatType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - MatType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } - //! Get the weight of the layer as a cube. - arma::Cube const& Weight() const - { - return weight; - } - //! Modify the weight of the layer as a cube. - arma::Cube& Weight() { return weight; } + //! Get the weight of the layer. + arma::cube const& Weight() const { return weight; } + //! Modify the weight of the layer. + arma::cube& Weight() { return weight; } //! Get the bias of the layer. - MatType const& Bias() const { return bias; } + arma::mat const& Bias() const { return bias; } //! Modify the bias of the layer. - MatType& Bias() { return bias; } + arma::mat& Bias() { return bias; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the input width. + size_t InputWidth() const { return inputWidth; } + //! Modify input the width. + size_t& InputWidth() { return inputWidth; } + + //! Get the input height. + size_t InputHeight() const { return inputHeight; } + //! Modify the input height. + size_t& InputHeight() { return inputHeight; } + + //! Get the output width. + size_t OutputWidth() const { return outputWidth; } + //! Modify the output width. + size_t& OutputWidth() { return outputWidth; } + + //! Get the output height. + size_t OutputHeight() const { return outputHeight; } + //! Modify the output height. + size_t& OutputHeight() { return outputHeight; } + + //! Get the number of input maps. + size_t InputSize() const { return inSize; } //! Get the number of output maps. - size_t const& Maps() const { return maps; } + size_t OutputSize() const { return outSize; } //! Get the kernel width. - size_t const& KernelWidth() const { return kernelWidth; } + size_t KernelWidth() const { return kernelWidth; } //! Modify the kernel width. size_t& KernelWidth() { return kernelWidth; } //! Get the kernel height. - size_t const& KernelHeight() const { return kernelHeight; } + size_t KernelHeight() const { return kernelHeight; } //! Modify the kernel height. size_t& KernelHeight() { return kernelHeight; } //! Get the stride width. - size_t const& StrideWidth() const { return strideWidth; } + size_t StrideWidth() const { return strideWidth; } //! Modify the stride width. size_t& StrideWidth() { return strideWidth; } //! Get the stride height. - size_t const& StrideHeight() const { return strideHeight; } + size_t StrideHeight() const { return strideHeight; } //! Modify the stride height. size_t& StrideHeight() { return strideHeight; } //! Get the top padding height. - size_t const& PadHTop() const { return padHTop; } + size_t PadHTop() const { return padHTop; } //! Modify the top padding height. size_t& PadHTop() { return padHTop; } //! Get the bottom padding height. - size_t const& PadHBottom() const { return padHBottom; } + size_t PadHBottom() const { return padHBottom; } //! Modify the bottom padding height. size_t& PadHBottom() { return padHBottom; } //! Get the left padding width. - size_t const& PadWLeft() const { return padWLeft; } + size_t PadWLeft() const { return padWLeft; } //! Modify the left padding width. size_t& PadWLeft() { return padWLeft; } //! Get the right padding width. - size_t const& PadWRight() const { return padWRight; } + size_t PadWRight() const { return padWRight; } //! Modify the right padding width. size_t& PadWRight() { return padWRight; } //! Get size of weights for the layer. size_t WeightSize() const { - return (maps * inMaps * higherInDimensions * kernelWidth * kernelHeight) + - maps; + return (outSize * inSize * kernelWidth * kernelHeight) + outSize; } - //! Compute the output dimensions of the layer based on `InputDimensions()`. - void ComputeOutputDimensions(); + //! Get the shape of the input. + size_t InputShape() const + { + return inputHeight * inputWidth * inSize; + } /** * Serialize the layer. @@ -263,7 +313,7 @@ class ConvolutionType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: - /** + /* * Return the convolution output size. * * @param size The size of the input (row or column). @@ -282,12 +332,12 @@ class ConvolutionType : public Layer return std::floor(size + pSideOne + pSideTwo - k) / s + 1; } - /** + /* * Function to assign padding such that output size is same as input size. */ void InitializeSamePadding(); - /** + /* * Rotates a 3rd-order tensor counterclockwise by 180 degrees. * * @param input The input data to be rotated. @@ -303,7 +353,7 @@ class ConvolutionType : public Layer output.slice(s) = arma::fliplr(arma::flipud(input.slice(s))); } - /** + /* * Rotates a dense matrix counterclockwise by 180 degrees. * * @param input The input data to be rotated. @@ -316,8 +366,11 @@ class ConvolutionType : public Layer output = arma::fliplr(arma::flipud(input)); } + //! Locally-stored number of input channels. + size_t inSize; + //! Locally-stored number of output channels. - size_t maps; + size_t outSize; //! Locally-stored number of input units. size_t batchSize; @@ -347,46 +400,54 @@ class ConvolutionType : public Layer size_t padHTop; //! Locally-stored weight object. - MatType weights; + OutputDataType weights; //! Locally-stored weight object. - arma::Cube weight; + arma::cube weight; //! Locally-stored bias term object. - MatType bias; + arma::mat bias; + + //! Locally-stored input width. + size_t inputWidth; + + //! Locally-stored input height. + size_t inputHeight; + + //! Locally-stored output width. + size_t outputWidth; + + //! Locally-stored output height. + size_t outputHeight; //! Locally-stored transformed output parameter. - arma::Cube outputTemp; + arma::cube outputTemp; //! Locally-stored transformed padded input parameter. - MatType inputPadded; + arma::cube inputPaddedTemp; //! Locally-stored transformed error parameter. - arma::Cube gTemp; + arma::cube gTemp; //! Locally-stored transformed gradient parameter. - arma::Cube gradientTemp; + arma::cube gradientTemp; //! Locally-stored padding layer. - ann::Padding padding; + ann::Padding<> padding; - //! Type of padding. - std::string paddingType; + //! Locally-stored delta object. + OutputDataType delta; - //! Locally-cached number of input maps. - size_t inMaps; - //! Locally-cached higher-order input dimensions. - size_t higherInDimensions; + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; }; // class Convolution -// Standard Convolution layer. -typedef ConvolutionType< - NaiveConvolution, - NaiveConvolution, - NaiveConvolution, - arma::mat -> Convolution; - } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index 5e4f2c618e..ba957b6ead 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -22,14 +22,16 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename MatType + typename InputDataType, + typename OutputDataType > -ConvolutionType< +Convolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - MatType ->::ConvolutionType() + InputDataType, + OutputDataType +>::Convolution() { // Nothing to do here. } @@ -38,30 +40,38 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename MatType + typename InputDataType, + typename OutputDataType > -ConvolutionType< +Convolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - MatType ->::ConvolutionType( - const size_t maps, + InputDataType, + OutputDataType +>::Convolution( + const size_t inSize, + const size_t outSize, const size_t kernelWidth, const size_t kernelHeight, const size_t strideWidth, const size_t strideHeight, const size_t padW, const size_t padH, + const size_t inputWidth, + const size_t inputHeight, const std::string& paddingType) : - ConvolutionType( - maps, + Convolution( + inSize, + outSize, kernelWidth, kernelHeight, strideWidth, strideHeight, std::tuple(padW, padW), std::tuple(padH, padH), + inputWidth, + inputHeight, paddingType) { // Nothing to do here. @@ -71,23 +81,29 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename MatType + typename InputDataType, + typename OutputDataType > -ConvolutionType< +Convolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - MatType ->::ConvolutionType( - const size_t maps, + InputDataType, + OutputDataType +>::Convolution( + const size_t inSize, + const size_t outSize, const size_t kernelWidth, const size_t kernelHeight, const size_t strideWidth, const size_t strideHeight, const std::tuple& padW, const std::tuple& padH, - const std::string& paddingTypeIn) : - maps(maps), + const size_t inputWidth, + const size_t inputHeight, + const std::string& paddingType) : + inSize(inSize), + outSize(outSize), kernelWidth(kernelWidth), kernelHeight(kernelHeight), strideWidth(strideWidth), @@ -95,484 +111,443 @@ ConvolutionType< padWLeft(std::get<0>(padW)), padWRight(std::get<1>(padW)), padHBottom(std::get<1>(padH)), - padHTop(std::get<0>(padH)) + padHTop(std::get<0>(padH)), + inputWidth(inputWidth), + inputHeight(inputHeight), + outputWidth(0), + outputHeight(0) { + weights.set_size(WeightSize(), 1); + // Transform paddingType to lowercase. - this->paddingType = util::ToLower(paddingTypeIn); -} + const std::string paddingTypeLow = util::ToLower(paddingType); -template< - typename ForwardConvolutionRule, - typename BackwardConvolutionRule, - typename GradientConvolutionRule, - typename MatType -> -ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->::ConvolutionType(const ConvolutionType& other) : - Layer(other), - maps(other.maps), - kernelWidth(other.kernelWidth), - kernelHeight(other.kernelHeight), - strideWidth(other.strideWidth), - strideHeight(other.strideHeight), - padWLeft(other.padWLeft), - padWRight(other.padWRight), - padHBottom(other.padHBottom), - padHTop(other.padHTop), - padding(other.padding), - paddingType(other.paddingType), - inMaps(other.inMaps), - higherInDimensions(other.higherInDimensions) -{ - // Nothing to do. -} - -template< - typename ForwardConvolutionRule, - typename BackwardConvolutionRule, - typename GradientConvolutionRule, - typename MatType -> -ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->::ConvolutionType(ConvolutionType&& other) : - Layer(std::move(other)), - maps(std::move(other.maps)), - kernelWidth(std::move(other.kernelWidth)), - kernelHeight(std::move(other.kernelHeight)), - strideWidth(std::move(other.strideWidth)), - strideHeight(std::move(other.strideHeight)), - padWLeft(std::move(other.padWLeft)), - padWRight(std::move(other.padWRight)), - padHBottom(std::move(other.padHBottom)), - padHTop(std::move(other.padHTop)), - padding(std::move(other.padding)), - paddingType(std::move(other.paddingType)), - inMaps(std::move(other.inMaps)), - higherInDimensions(std::move(other.higherInDimensions)) -{ - // Nothing to do. -} - -template< - typename ForwardConvolutionRule, - typename BackwardConvolutionRule, - typename GradientConvolutionRule, - typename MatType -> -ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->& -ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->::operator=(const ConvolutionType& other) -{ - if (&other != this) - { - Layer::operator=(other); - maps = other.maps; - kernelWidth = other.kernelWidth; - kernelHeight = other.kernelHeight; - strideWidth = other.strideWidth; - strideHeight = other.strideHeight; - padWLeft = other.padWLeft; - padWRight = other.padWRight; - padHBottom = other.padHBottom; - padHTop = other.padHTop; - padding = other.padding; - paddingType = other.paddingType; - inMaps = other.inMaps; - higherInDimensions = other.higherInDimensions; - } - - return *this; -} - -template< - typename ForwardConvolutionRule, - typename BackwardConvolutionRule, - typename GradientConvolutionRule, - typename MatType -> -ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->& -ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->::operator=(ConvolutionType&& other) -{ - if (&other != this) - { - Layer::operator=(std::move(other)); - maps = std::move(other.maps); - kernelWidth = std::move(other.kernelWidth); - kernelHeight = std::move(other.kernelHeight); - strideWidth = std::move(other.strideWidth); - strideHeight = std::move(other.strideHeight); - padWLeft = std::move(other.padWLeft); - padWRight = std::move(other.padWRight); - padHBottom = std::move(other.padHBottom); - padHTop = std::move(other.padHTop); - padding = std::move(other.padding); - paddingType = std::move(other.paddingType); - inMaps = std::move(other.inMaps); - higherInDimensions = std::move(other.higherInDimensions); - } - - return *this; -} - -template< - typename ForwardConvolutionRule, - typename BackwardConvolutionRule, - typename GradientConvolutionRule, - typename MatType -> -void ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->::SetWeights(typename MatType::elem_type* weightPtr) -{ - MakeAlias(weight, weightPtr, kernelWidth, kernelHeight, maps * inMaps); - MakeAlias(bias, weightPtr + weight.n_elem, maps, 1); - MakeAlias(weights, weightPtr, weight.n_elem + bias.n_elem, 1); -} - -template< - typename ForwardConvolutionRule, - typename BackwardConvolutionRule, - typename GradientConvolutionRule, - typename MatType -> -void ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->::Forward(const MatType& input, MatType& output) -{ - batchSize = input.n_cols; - - // First, perform any padding if necessary. - const bool usingPadding = - (padWLeft != 0 || padWRight != 0 || padHTop != 0 || padHBottom != 0); - const size_t paddedRows = this->inputDimensions[0] + padWLeft + padWRight; - const size_t paddedCols = this->inputDimensions[1] + padHTop + padHBottom; - if (usingPadding) - { - inputPadded.set_size(paddedRows * paddedCols * inMaps * higherInDimensions, - input.n_cols); - padding.Forward(input, inputPadded); - } - - arma::Cube inputTemp; - MakeAlias(inputTemp, - const_cast(usingPadding ? inputPadded : input).memptr(), - paddedRows, paddedCols, inMaps * higherInDimensions * batchSize); - - MakeAlias(outputTemp, output.memptr(), this->outputDimensions[0], - this->outputDimensions[1], maps * higherInDimensions * batchSize); - outputTemp.zeros(); - - // We "ignore" dimensions higher than the third---that means that we just pass - // them through and treat them like different input points. - // - // If we eventually have a way to do convolutions for a single kernel - // in-batch, then this strategy may not be the most efficient solution. - for (size_t offset = 0; offset < (higherInDimensions * batchSize); ++offset) - { - const size_t fullInputOffset = offset * inMaps; - const size_t fullOutputOffset = offset * maps; - - // Iterate over output maps. - for (size_t outMap = 0; outMap < maps; ++outMap) - { - // Iterate over input maps (we will apply the filter and sum). - for (size_t inMap = 0; inMap < inMaps; ++inMap) - { - MatType convOutput; - - ForwardConvolutionRule::Convolution( - inputTemp.slice(inMap + fullInputOffset), - weight.slice(outMap), - convOutput, - strideWidth, - strideHeight); - - outputTemp.slice(outMap + fullOutputOffset) += convOutput; - } - - // Make sure to add the bias. - outputTemp.slice(outMap + fullOutputOffset) += bias(outMap); - } - } -} - -template< - typename ForwardConvolutionRule, - typename BackwardConvolutionRule, - typename GradientConvolutionRule, - typename MatType -> -void ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->::Backward( - const MatType& /* input */, const MatType& gy, MatType& g) -{ - arma::Cube mappedError; - MakeAlias(mappedError, ((MatType&) gy).memptr(), this->outputDimensions[0], - this->outputDimensions[1], higherInDimensions * maps * batchSize); - - MakeAlias(gTemp, g.memptr(), this->inputDimensions[0], - this->inputDimensions[1], inMaps * higherInDimensions * batchSize); - gTemp.zeros(); - - const bool usingPadding = - (padWLeft != 0 || padWRight != 0 || padHTop != 0 || padHBottom != 0); - - // To perform the backward pass, we need to rotate all the filters. - arma::Cube rotatedFilters(weight.n_cols, - weight.n_rows, weight.n_slices); - for (size_t map = 0; map < maps; ++map) - { - Rotate180(weight.slice(map), rotatedFilters.slice(map)); - } - - // See Forward() for the overall iteration strategy. - for (size_t offset = 0; offset < (higherInDimensions * batchSize); ++offset) - { - const size_t fullInputOffset = offset * inMaps; - const size_t fullOutputOffset = offset * maps; - - // Iterate over input maps. - for (size_t inMap = 0; inMap < inMaps; ++inMap) - { - // Iterate over output maps. - for (size_t outMap = 0; outMap < maps; ++outMap) - { - MatType output; - - BackwardConvolutionRule::Convolution( - mappedError.slice(outMap + fullOutputOffset), - rotatedFilters.slice(outMap), - output, - strideHeight, - strideWidth); - - // If the stride width or height is greater than 1, then we have to - // insert columns and rows into the convolution output. - if (strideWidth == 1 && strideHeight == 1) - { - if (usingPadding) - { - gTemp.slice(inMap + fullInputOffset) += output.submat( - padWLeft, - padHTop, - padWLeft + gTemp.n_rows - 1, - padHTop + gTemp.n_cols - 1); - } - else - { - gTemp.slice(inMap + fullInputOffset) += output; - } - } - else - { - // We must iterate over each element of the output and manually - // re-insert the stride. - size_t col = padWLeft; - for (size_t i = 0; i < output.n_cols; ++i) - { - size_t row = padHTop; - for (size_t j = 0; j < output.n_rows; ++j) - { - gTemp(row, col, inMap + fullInputOffset) += output(j, i); - row += strideHeight; - } - col += strideWidth; - } - } - } - } - } -} - -template< - typename ForwardConvolutionRule, - typename BackwardConvolutionRule, - typename GradientConvolutionRule, - typename MatType -> -void ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->::Gradient( - const MatType& input, - const MatType& error, - MatType& gradient) -{ - arma::Cube mappedError; - MakeAlias(mappedError, ((MatType&) error).memptr(), - this->outputDimensions[0], this->outputDimensions[1], - higherInDimensions * maps * batchSize); - - // We are depending here on `inputPadded` being properly set from a call to - // Forward(). - const bool usingPadding = - (padWLeft != 0 || padWRight != 0 || padHTop != 0 || padHBottom != 0); - const size_t paddedRows = this->inputDimensions[0] + padWLeft + padWRight; - const size_t paddedCols = this->inputDimensions[1] + padHTop + padHBottom; - - arma::Cube inputTemp( - const_cast(usingPadding ? inputPadded : input).memptr(), - paddedRows, paddedCols, inMaps * batchSize, false, false); - - // We will make an alias for the gradient, but note that this is only for the - // convolution map weights! The bias will be handled by direct accesses into - // `gradient`. - gradient.zeros(); - MakeAlias(gradientTemp, gradient.memptr(), weight.n_rows, weight.n_cols, - weight.n_slices); - - // See Forward() for our iteration strategy. - for (size_t offset = 0; offset < higherInDimensions * batchSize; ++offset) - { - const size_t fullInputOffset = offset * inMaps; - const size_t fullOutputOffset = offset * maps; - - for (size_t outMap = 0; outMap < maps; ++outMap) - { - for (size_t inMap = 0; inMap < inMaps; ++inMap) - { - MatType output; - GradientConvolutionRule::Convolution( - inputTemp.slice(inMap + fullInputOffset), - mappedError.slice(outMap + fullOutputOffset), - output, - strideWidth, - strideHeight); - - // TODO: understand this conditional. Is it needed? - if (gradientTemp.n_rows < output.n_rows || - gradientTemp.n_cols < output.n_cols) - { - gradientTemp.slice(outMap) += output.submat(0, 0, - gradientTemp.n_rows - 1, gradientTemp.n_cols - 1); - } - else if (gradientTemp.n_rows > output.n_rows || - gradientTemp.n_cols > output.n_cols) - { - gradientTemp.slice(outMap).submat(0, 0, output.n_rows - 1, - output.n_cols - 1) += output; - } - else - { - gradientTemp.slice(outMap) += output; - } - } - - gradient[weight.n_elem + outMap] += arma::accu(mappedError.slice(outMap + - fullOutputOffset)); - } - } -} - -template< - typename ForwardConvolutionRule, - typename BackwardConvolutionRule, - typename GradientConvolutionRule, - typename MatType -> -void ConvolutionType< - ForwardConvolutionRule, - BackwardConvolutionRule, - GradientConvolutionRule, - MatType ->::ComputeOutputDimensions() -{ - // First, we must make sure the padding sizes are up to date, which we can - // now do since inputDimensions is set correctly. - if (paddingType == "valid") + if (paddingTypeLow == "valid") { padWLeft = 0; padWRight = 0; padHTop = 0; padHBottom = 0; } - else if (paddingType == "same") + else if (paddingTypeLow == "same") { InitializeSamePadding(); } - padding = ann::Padding(padWLeft, padWRight, padHTop, padHBottom); - padding.InputDimensions() = this->inputDimensions; - padding.ComputeOutputDimensions(); - - // We must ensure that the output has at least 3 dimensions, since we will - // be adding some number of maps to the output. - this->outputDimensions = std::vector( - std::max(this->inputDimensions.size(), size_t(3)), 1); - this->outputDimensions[0] = ConvOutSize(this->inputDimensions[0], - kernelWidth, strideWidth, padWLeft, padWRight); - this->outputDimensions[1] = ConvOutSize(this->inputDimensions[1], - kernelHeight, strideHeight, padHTop, padHBottom); - - inMaps = (this->inputDimensions.size() >= 3) ? this->inputDimensions[2] : 1; - - // Compute and cache the total number of input maps. - higherInDimensions = 1; - for (size_t i = 3; i < this->inputDimensions.size(); ++i) - { - higherInDimensions *= this->inputDimensions[i]; - this->outputDimensions[i] = this->inputDimensions[i]; - } - - this->outputDimensions[2] = maps; + padding = ann::Padding<>(padWLeft, padWRight, padHTop, padHBottom); } template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename MatType + typename InputDataType, + typename OutputDataType > -template -void ConvolutionType< +Convolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - MatType + InputDataType, + OutputDataType +>::Convolution( + const Convolution& layer) : + inSize(layer.inSize), + outSize(layer.outSize), + kernelWidth(layer.kernelWidth), + kernelHeight(layer.kernelHeight), + strideWidth(layer.strideWidth), + strideHeight(layer.strideHeight), + padWLeft(layer.padWLeft), + padWRight(layer.padWRight), + padHBottom(layer.padHBottom), + padHTop(layer.padHTop), + weights(layer.weights), + inputWidth(layer.inputWidth), + inputHeight(layer.inputHeight), + outputWidth(layer.outputWidth), + outputHeight(layer.outputHeight), + padding(layer.padding) +{ + // Nothing to do here. +} + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>::Convolution( + Convolution&& layer) : + inSize(0), + outSize(0), + kernelWidth(layer.kernelWidth), + kernelHeight(layer.kernelHeight), + strideWidth(layer.strideWidth), + strideHeight(layer.strideHeight), + padWLeft(layer.padWLeft), + padWRight(layer.padWRight), + padHBottom(layer.padHBottom), + padHTop(layer.padHTop), + weights(std::move(layer.weights)), + inputWidth(layer.inputWidth), + inputHeight(layer.inputHeight), + outputWidth(layer.outputWidth), + outputHeight(layer.outputHeight), + padding(std::move(layer.padding)) +{ + // Nothing to do here. +} + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>& +Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>:: +operator=(const Convolution& layer) +{ + if (this != &layer) + { + inSize = layer.inSize; + outSize = layer.outSize; + kernelWidth = layer.kernelWidth; + kernelHeight = layer.kernelHeight; + strideWidth = layer.strideWidth; + strideHeight = layer.strideHeight; + padWLeft = layer.padWLeft; + padWRight = layer.padWRight; + padHBottom = layer.padHBottom; + padHTop = layer.padHTop; + inputWidth = layer.inputWidth; + inputHeight = layer.inputHeight; + outputWidth = layer.outputWidth; + outputHeight = layer.outputHeight; + padding = layer.padding; + weights = layer.weights; + } + + return *this; +} +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>& +Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>:: +operator=(Convolution&& layer) +{ + if (this != &layer) + { + inSize = layer.inSize; + outSize = layer.outSize; + kernelWidth = layer.kernelWidth; + kernelHeight = layer.kernelHeight; + strideWidth = layer.strideWidth; + strideHeight = layer.strideHeight; + padWLeft = layer.padWLeft; + padWRight = layer.padWRight; + padHBottom = layer.padHBottom; + padHTop = layer.padHTop; + inputWidth = layer.inputWidth; + inputHeight = layer.inputHeight; + outputWidth = layer.outputWidth; + outputHeight = layer.outputHeight; + padding = std::move(layer.padding); + weights = std::move(layer.weights); + } + + return *this; +} + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +void Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>::Reset() +{ + weight = arma::cube(weights.memptr(), kernelWidth, kernelHeight, + outSize * inSize, false, false); + bias = arma::mat(weights.memptr() + weight.n_elem, + outSize, 1, false, false); +} + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +template +void Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>::Forward(const arma::Mat& input, arma::Mat& output) +{ + batchSize = input.n_cols; + arma::cube inputTemp(const_cast&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); + + if (padWLeft != 0 || padWRight != 0 || padHTop != 0 || padHBottom != 0) + { + inputPaddedTemp.set_size(inputTemp.n_rows + padWLeft + padWRight, + inputTemp.n_cols + padHTop + padHBottom, inputTemp.n_slices); + + for (size_t i = 0; i < inputTemp.n_slices; ++i) + { + padding.Forward(inputTemp.slice(i), inputPaddedTemp.slice(i)); + } + } + + size_t wConv = ConvOutSize(inputWidth, kernelWidth, strideWidth, padWLeft, + padWRight); + size_t hConv = ConvOutSize(inputHeight, kernelHeight, strideHeight, padHTop, + padHBottom); + + output.set_size(wConv * hConv * outSize, batchSize); + outputTemp = arma::Cube(output.memptr(), wConv, hConv, + outSize * batchSize, false, false); + outputTemp.zeros(); + + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) + { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) + { + arma::Mat convOutput; + + if (padWLeft != 0 || padWRight != 0 || padHTop != 0 || padHBottom != 0) + { + ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, + strideWidth, strideHeight); + } + else + { + ForwardConvolutionRule::Convolution(inputTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, + strideWidth, strideHeight); + } + + outputTemp.slice(outMap) += convOutput; + } + + outputTemp.slice(outMap) += bias(outMap % outSize); + } + + outputWidth = outputTemp.n_rows; + outputHeight = outputTemp.n_cols; +} + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +template +void Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) +{ + arma::cube mappedError(((arma::Mat&) gy).memptr(), outputWidth, + outputHeight, outSize * batchSize, false, false); + + g.set_size(inputWidth * inputHeight * inSize, batchSize); + gTemp = arma::Cube(g.memptr(), inputWidth, inputHeight, + inSize * batchSize, false, false); + gTemp.zeros(); + + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) + { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) + { + arma::Mat output, rotatedFilter; + Rotate180(weight.slice(outMapIdx), rotatedFilter); + + BackwardConvolutionRule::Convolution(mappedError.slice(outMap), + rotatedFilter, output, strideWidth, strideHeight); + + if (padWLeft != 0 || padWRight != 0 || padHTop != 0 || padHBottom != 0) + { + gTemp.slice(inMap + batchCount * inSize) += output.submat(padWLeft, + padHTop, padWLeft + gTemp.n_rows - 1, padHTop + gTemp.n_cols - 1); + } + else + { + gTemp.slice(inMap + batchCount * inSize) += output; + } + } + } +} + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +template +void Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType +>::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) +{ + arma::cube mappedError(((arma::Mat&) error).memptr(), outputWidth, + outputHeight, outSize * batchSize, false, false); + arma::cube inputTemp(((arma::Mat&) input).memptr(), inputWidth, + inputHeight, inSize * batchSize, false, false); + + gradient.set_size(weights.n_elem, 1); + gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, + weight.n_cols, weight.n_slices, false, false); + gradientTemp.zeros(); + + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) + { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) + { + arma::Mat inputSlice; + if (padWLeft != 0 || padWRight != 0 || padHTop != 0 || padHBottom != 0) + { + inputSlice = inputPaddedTemp.slice(inMap + batchCount * inSize); + } + else + { + inputSlice = inputTemp.slice(inMap + batchCount * inSize); + } + + arma::Mat deltaSlice = mappedError.slice(outMap); + + arma::Mat output; + GradientConvolutionRule::Convolution(inputSlice, deltaSlice, + output, strideWidth, strideHeight); + + if (gradientTemp.n_rows < output.n_rows || + gradientTemp.n_cols < output.n_cols) + { + gradientTemp.slice(outMapIdx) += output.submat(0, 0, + gradientTemp.n_rows - 1, gradientTemp.n_cols - 1); + } + else if (gradientTemp.n_rows > output.n_rows || + gradientTemp.n_cols > output.n_cols) + { + gradientTemp.slice(outMapIdx).submat(0, 0, output.n_rows - 1, + output.n_cols - 1) += output; + } + else + { + gradientTemp.slice(outMapIdx) += output; + } + } + + gradient.submat(weight.n_elem + (outMap % outSize), 0, weight.n_elem + + (outMap % outSize), 0) = arma::accu(mappedError.slice(outMap)); + } +} + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +template +void Convolution< + ForwardConvolutionRule, + BackwardConvolutionRule, + GradientConvolutionRule, + InputDataType, + OutputDataType >::serialize(Archive& ar, const uint32_t /* version*/) { - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(maps)); + ar(CEREAL_NVP(inSize)); + ar(CEREAL_NVP(outSize)); ar(CEREAL_NVP(batchSize)); ar(CEREAL_NVP(kernelWidth)); ar(CEREAL_NVP(kernelHeight)); @@ -582,31 +557,41 @@ void ConvolutionType< ar(CEREAL_NVP(padWRight)); ar(CEREAL_NVP(padHBottom)); ar(CEREAL_NVP(padHTop)); + ar(CEREAL_NVP(inputWidth)); + ar(CEREAL_NVP(inputHeight)); + ar(CEREAL_NVP(outputWidth)); + ar(CEREAL_NVP(outputHeight)); ar(CEREAL_NVP(padding)); - ar(CEREAL_NVP(inMaps)); - ar(CEREAL_NVP(higherInDimensions)); + + if (cereal::is_loading()) + { + weights.set_size((outSize * inSize * kernelWidth * kernelHeight) + outSize, + 1); + } } template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename MatType + typename InputDataType, + typename OutputDataType > -void ConvolutionType< +void Convolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - MatType + InputDataType, + OutputDataType >::InitializeSamePadding() { /* * Using O = (W - F + 2P) / s + 1; */ - size_t totalVerticalPadding = (strideWidth - 1) * this->inputDimensions[0] + - kernelWidth - strideWidth; - size_t totalHorizontalPadding = (strideHeight - 1) * this->inputDimensions[1] - + kernelHeight - strideHeight; + size_t totalVerticalPadding = (strideWidth - 1) * inputWidth + kernelWidth - + strideWidth; + size_t totalHorizontalPadding = (strideHeight - 1) * inputHeight + + kernelHeight - strideHeight; padWLeft = totalVerticalPadding / 2; padWRight = totalVerticalPadding - totalVerticalPadding / 2; diff --git a/src/mlpack/methods/ann/layer/dropconnect.hpp b/src/mlpack/methods/ann/layer/dropconnect.hpp index c00a2d2129..db8d76aa4b 100644 --- a/src/mlpack/methods/ann/layer/dropconnect.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect.hpp @@ -16,7 +16,10 @@ #include -#include "layer.hpp" +#include "layer_types.hpp" +#include "add_merge.hpp" +#include "linear.hpp" +#include "sequential.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -25,58 +28,55 @@ namespace ann /** Artificial Neural Network. */ { * The DropConnect layer is a regularizer that randomly with probability * ratio sets the connection values to zero and scales the remaining * elements by factor 1 /(1 - ratio). The output is scaled with 1 / (1 - p) - * when in training mode. During testing, the layer just computes the output. - * The output is computed according to the input layer. If no input layer is - * given, it will take a linear layer as default. + * when deterministic is false. In the deterministic mode(during testing), + * the layer just computes the output. The output is computed according + * to the input layer. If no input layer is given, it will take a linear layer + * as default. * - * For more information, see the following. + * Note: + * During training you should set deterministic to false and during testing + * you should set deterministic to true. + * + * For more information, see the following. * * @code * @inproceedings{WanICML2013, - * title = {Regularization of Neural Networks using DropConnect}, + * title={Regularization of Neural Networks using DropConnect}, * booktitle = {Proceedings of the 30th International Conference on Machine * Learning(ICML - 13)}, - * author = {Li Wan and Matthew Zeiler and Sixin Zhang and Yann L. Cun and - * Rob Fergus}, - * year = {2013}, - * url = {http://proceedings.mlr.press/v28/wan13.pdf} + * author = {Li Wan and Matthew Zeiler and Sixin Zhang and Yann L. Cun and + * Rob Fergus}, + * year = {2013}, + * url = {http://proceedings.mlr.press/v28/wan13.pdf} * } * @endcode * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class DropConnectType : public Layer +template< + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class DropConnect { public: //! Create the DropConnect object. - DropConnectType(); + DropConnect(); /** - * Creates the DropConnect Layer as a Linear Object that takes the number of - * output units and a ratio as parameter. + * Creates the DropConnect Layer as a Linear Object that takes input size, + * output size and ratio as parameter. * + * @param inSize The number of input units. * @param outSize The number of output units. * @param ratio The probability of setting a value to zero. */ - DropConnectType(const size_t outSize, - const double ratio = 0.5); - - //! Clone the DropConnectType object. This handles polymorphism correctly. - DropConnectType* Clone() const { return new DropConnectType(*this); } - - // Virtual destructor. - virtual ~DropConnectType(); - - //! Copy the given DropConnectType (except for weights). - DropConnectType(const DropConnectType& other); - //! Take ownership of the given DropConnectType (except for weights). - DropConnectType(DropConnectType&& other); - //! Copy the given DropConnectType (except for weights). - DropConnectType& operator=(const DropConnectType& other); - //! Take ownership of the given DropConnectType (except for weights). - DropConnectType& operator=(DropConnectType&& other); + DropConnect(const size_t inSize, + const size_t outSize, + const double ratio = 0.5); /** * Ordinary feed forward pass of the DropConnect layer. @@ -84,7 +84,8 @@ class DropConnectType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the DropConnect layer. @@ -93,7 +94,10 @@ class DropConnectType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& input, const MatType& gy, MatType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activation. @@ -102,7 +106,39 @@ class DropConnectType : public Layer * @param error The calculated error. * @param * (gradient) The calculated gradient. */ - void Gradient(const MatType& input, const MatType& error, MatType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */); + + //! Get the model modules. + std::vector >& Model() { return network; } + + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! The value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + + //! Modify the value of the deterministic parameter. + bool &Deterministic() { return deterministic; } //! The probability of setting a value to zero. double Ratio() const { return ratio; } @@ -114,14 +150,8 @@ class DropConnectType : public Layer scale = 1.0 / (1.0 - ratio); } - //! Compute the output dimensions of the layer based on `InputDimensions()`. - void ComputeOutputDimensions(); - - //! Return the size of the weights. - size_t WeightSize() const { return baseLayer->WeightSize(); } - - // Set the weights to use the given memory `weightsPtr`. - void SetWeights(typename MatType::elem_type* weightsPtr); + //! Return the size of the weight matrix. + size_t WeightSize() const { return 0; } /** * Serialize the layer. @@ -136,21 +166,34 @@ class DropConnectType : public Layer //! The scale fraction. double scale; + //! Locally-stored weight object. + OutputDataType weights; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Locally-stored mask object. - MatType mask; + OutputDataType mask; + + //! If true dropout and scaling is disabled, see notes above. + bool deterministic; //! Denoise mask for the weights. - MatType denoise; + OutputDataType denoise; //! Locally-stored layer module. - Layer* baseLayer; + LayerTypes<> baseLayer; + + //! Locally-stored network modules. + std::vector > network; }; // class DropConnect. -// Convenience typedefs. - -// Standard DropConnect layer. -typedef DropConnectType DropConnect; - } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/dropconnect_impl.hpp b/src/mlpack/methods/ann/layer/dropconnect_impl.hpp index 9eb8bb91a4..9b22c91951 100644 --- a/src/mlpack/methods/ann/layer/dropconnect_impl.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect_impl.hpp @@ -17,160 +17,112 @@ // In case it hasn't yet been included. #include "dropconnect.hpp" -#include "linear.hpp" +#include "../visitor/delete_visitor.hpp" +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" +#include "../visitor/parameters_set_visitor.hpp" +#include "../visitor/parameters_visitor.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -DropConnectType::DropConnectType() : - Layer(), +template +DropConnect::DropConnect() : ratio(0.5), scale(2.0), - baseLayer(new LinearType(0)) + deterministic(true) { // Nothing to do here. } -template -DropConnectType::DropConnectType( +template +DropConnect::DropConnect( + const size_t inSize, const size_t outSize, const double ratio) : - Layer(), ratio(ratio), scale(1.0 / (1 - ratio)), - baseLayer(new LinearType(outSize)) + baseLayer(new Linear(inSize, outSize)) { - // Nothing to do. + network.push_back(baseLayer); } -template -DropConnectType::~DropConnectType() +template +template +void DropConnect::Forward( + const arma::Mat& input, + arma::Mat& output) { - delete baseLayer; -} - -template -DropConnectType::DropConnectType(const DropConnectType& other) : - Layer(other), - ratio(other.ratio), - scale(other.scale), - baseLayer(other.baseLayer->Clone()) -{ - // Nothing to do. -} - -template -DropConnectType::DropConnectType(DropConnectType&& other) : - Layer(std::move(other)), - ratio(std::move(other.ratio)), - scale(std::move(other.scale)), - baseLayer(std::move(other.baseLayer)) -{ - // Nothing to do. -} - -template -DropConnectType& -DropConnectType::operator=(const DropConnectType& other) -{ - if (&other != this) + // The DropConnect mask will not be multiplied in the deterministic mode + // (during testing). + if (deterministic) { - Layer::operator=(other); - ratio = other.ratio; - scale = other.scale; - baseLayer = other.baseLayer->Clone(); - } - - return *this; -} - -template -DropConnectType& -DropConnectType::operator=(DropConnectType&& other) -{ - if (&other != this) - { - Layer::operator=(std::move(other)); - ratio = std::move(other.ratio); - scale = std::move(other.scale); - baseLayer = std::move(other.baseLayer); - } - - return *this; -} - -template -void DropConnectType::Forward(const MatType& input, MatType& output) -{ - // The DropConnect mask will not be multiplied in testing mode. - if (!this->training) - { - baseLayer->Forward(input, output); + boost::apply_visitor(ForwardVisitor(input, output), baseLayer); } else { // Save weights for denoising. - denoise = baseLayer->Parameters(); + boost::apply_visitor(ParametersVisitor(denoise), baseLayer); // Scale with input / (1 - ratio) and set values to zero with // probability ratio. - mask = arma::randu(denoise.n_rows, denoise.n_cols); + mask = arma::randu >(denoise.n_rows, denoise.n_cols); mask.transform([&](double val) { return (val > ratio); }); - baseLayer->Parameters() = denoise % mask; - baseLayer->Forward(input, output); + arma::mat tmp = denoise % mask; + boost::apply_visitor(ParametersSetVisitor(tmp), baseLayer); + + boost::apply_visitor(ForwardVisitor(input, output), baseLayer); output = output * scale; } } -template -void DropConnectType::Backward( - const MatType& input, - const MatType& gy, - MatType& g) +template +template +void DropConnect::Backward( + const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { - baseLayer->Backward(input, gy, g); + boost::apply_visitor(BackwardVisitor(input, gy, g), baseLayer); } -template -void DropConnectType::Gradient( - const MatType& input, - const MatType& error, - MatType& gradient) +template +template +void DropConnect::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */) { - baseLayer->Gradient(input, error, gradient); + boost::apply_visitor(GradientVisitor(input, error), + baseLayer); // Denoise the weights. - baseLayer->Parameters() = denoise; + boost::apply_visitor(ParametersSetVisitor(denoise), baseLayer); } -template -void DropConnectType::ComputeOutputDimensions() -{ - // Propagate input dimensions to the base layer. - baseLayer->InputDimensions() = this->inputDimensions; - this->outputDimensions = baseLayer->OutputDimensions(); -} - -template -void DropConnectType::SetWeights( - typename MatType::elem_type* weightsPtr) -{ - baseLayer->SetWeights(weightsPtr); -} - -template +template template -void DropConnectType::serialize( +void DropConnect::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); + // Delete the old network first, if needed. + if (cereal::is_loading()) + { + boost::apply_visitor(DeleteVisitor(), baseLayer); + } ar(CEREAL_NVP(ratio)); ar(CEREAL_NVP(scale)); - ar(CEREAL_POINTER(baseLayer)); + ar(CEREAL_VARIANT_POINTER(baseLayer)); + + if (cereal::is_loading()) + { + network.clear(); + network.push_back(baseLayer); + } } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp index f03250145f..f3fb8f18b4 100644 --- a/src/mlpack/methods/ann/layer/dropout.hpp +++ b/src/mlpack/methods/ann/layer/dropout.hpp @@ -15,16 +15,18 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { + /** * The dropout layer is a regularizer that randomly with probability 'ratio' * sets input values to zero and scales the remaining elements by factor 1 / * (1 - ratio) rather than during test time so as to keep the expected sum same. - * When the layer is in testing mode, there is no change in the input. + * In the deterministic mode (during testing), there is no change in the input. + * + * Note: During training you should set deterministic to false and during + * testing you should set deterministic to true. * * For more information, see the following. * @@ -41,11 +43,14 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class DropoutType : public Layer +template +class Dropout { public: /** @@ -53,22 +58,19 @@ class DropoutType : public Layer * * @param ratio The probability of setting a value to zero. */ - DropoutType(const double ratio = 0.5); + Dropout(const double ratio = 0.5); - //! Clone the DropoutType object. This handles polymorphism correctly. - DropoutType* Clone() const { return new DropoutType(*this); } + //! Copy Constructor + Dropout(const Dropout& layer); - // Virtual destructor. - virtual ~DropoutType() { } + //! Move Constructor + Dropout(const Dropout&&); - //! Copy the given DropoutType. - DropoutType(const DropoutType& other); - //! Take ownership of the given DropoutType. - DropoutType(DropoutType&& other); - //! Copy the given DropoutType. - DropoutType& operator=(const DropoutType& other); - //! Take ownership of the given DropoutType. - DropoutType& operator=(DropoutType&& other); + //! Copy assignment operator + Dropout& operator=(const Dropout& layer); + + //! Move assignment operator + Dropout& operator=(Dropout&& layer); /** * Ordinary feed forward pass of the dropout layer. @@ -76,7 +78,8 @@ class DropoutType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the dropout layer. @@ -85,7 +88,25 @@ class DropoutType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, const MatType& gy, MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the detla. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! The value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } //! The probability of setting a value to zero. double Ratio() const { return ratio; } @@ -104,20 +125,24 @@ class DropoutType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: - //! Locally-stored mask object. - MatType mask; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored mast object. + OutputDataType mask; //! The probability of setting a value to zero. double ratio; //! The scale fraction. double scale; -}; // class DropoutType -// Convenience typedefs. - -// Standard Dropout layer. -typedef DropoutType Dropout; + //! If true dropout and scaling is disabled, see notes above. + bool deterministic; +}; // class Dropout } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/dropout_impl.hpp b/src/mlpack/methods/ann/layer/dropout_impl.hpp index 012de36f62..80f0d81fec 100644 --- a/src/mlpack/methods/ann/layer/dropout_impl.hpp +++ b/src/mlpack/methods/ann/layer/dropout_impl.hpp @@ -19,66 +19,73 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -DropoutType::DropoutType( +template +Dropout::Dropout( const double ratio) : ratio(ratio), - scale(1.0 / (1.0 - ratio)) + scale(1.0 / (1.0 - ratio)), + deterministic(false) { // Nothing to do here. } -template -DropoutType::DropoutType(const DropoutType& other) : - Layer(other), - ratio(other.ratio), - scale(other.scale) +template +Dropout::Dropout( + const Dropout& layer) : + ratio(layer.ratio), + scale(layer.scale), + deterministic(layer.deterministic) { - // Nothing to do. + // Nothing to do here. } -template -DropoutType::DropoutType(DropoutType&& other) : - Layer(std::move(other)), - ratio(std::move(other.ratio)), - scale(std::move(other.scale)) +template +Dropout::Dropout( + const Dropout&& layer) : + ratio(std::move(layer.ratio)), + scale(std::move(scale)), + deterministic(std::move(deterministic)) { - // Nothing to do. + // Nothing to do here. } -template -DropoutType& -DropoutType::operator=(const DropoutType& other) +template +Dropout& +Dropout:: +operator=(const Dropout& layer) { - if (&other != this) + if (this != &layer) { - Layer::operator=(other); - ratio = other.ratio; - scale = other.scale; + ratio = layer.ratio; + scale = layer.scale; + deterministic = layer.deterministic; } - return *this; } -template -DropoutType& -DropoutType::operator=(DropoutType&& other) +template +Dropout& +Dropout:: +operator=(Dropout&& layer) { - if (&other != this) + if (this != &layer) { - Layer::operator=(std::move(other)); - ratio = std::move(other.ratio); - scale = std::move(other.scale); + ratio = std::move(layer.ratio); + scale = std::move(layer.scale); + deterministic = std::move(layer.deterministic); } - return *this; } -template -void DropoutType::Forward(const MatType& input, MatType& output) +template +template +void Dropout::Forward( + const arma::Mat& input, + arma::Mat& output) { - // The dropout mask will not be multiplied in testing mode. - if (!this->training) + // The dropout mask will not be multiplied in the deterministic mode + // (during testing). + if (deterministic) { output = input; } @@ -86,29 +93,28 @@ void DropoutType::Forward(const MatType& input, MatType& output) { // Scale with input / (1 - ratio) and set values to zero with probability // 'ratio'. - mask = arma::randu(input.n_rows, input.n_cols); + mask = arma::randu >(input.n_rows, input.n_cols); mask.transform([&](double val) { return (val > ratio); }); output = input % mask * scale; } } -template -void DropoutType::Backward( - const MatType& /* input */, - const MatType& gy, - MatType& g) +template +template +void Dropout::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { g = gy % mask * scale; } -template +template template -void DropoutType::serialize( +void Dropout::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(ratio)); // Reset scale. diff --git a/src/mlpack/methods/ann/layer/not_adapted/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp similarity index 70% rename from src/mlpack/methods/ann/layer/not_adapted/elu.hpp rename to src/mlpack/methods/ann/layer/elu.hpp index 292f61cc68..1b51455a93 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -6,9 +6,9 @@ * Definition of the ELU activation function as described by Djork-Arne Clevert, * Thomas Unterthiner and Sepp Hochreiter. * - * Definition of the SELU function as introduced by Klambauer et. al. in Self - * Neural Networks. The SELU activation function keeps the mean and variance of - * the input invariant. + * Definition of the SELU function as introduced by + * Klambauer et. al. in Self Neural Networks. The SELU activation + * function keeps the mean and variance of the input invariant. * * In short, SELU = lambda * ELU, with 'alpha' and 'lambda' fixed for * normalized inputs. @@ -26,8 +26,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -63,6 +61,7 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * + * * The SELU activation function is defined by * * @f{eqnarray*}{ @@ -93,19 +92,23 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * In testing mode, there is no computation of the derivative. + * In the deterministic mode, there is no computation of the derivative. * + * @note During training deterministic should be set to false and during + * testing/inference deterministic should be set to true. * @note Make sure to use SELU activation function with normalized inputs and * weights initialized with Lecun Normal Initialization. * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class ELUType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class ELU { public: /** @@ -113,7 +116,7 @@ class ELUType : public Layer * * NOTE: Use this constructor for SELU activation function. */ - ELUType(); + ELU(); /** * Create the ELU object using the specified parameter. The non zero @@ -123,10 +126,8 @@ class ELUType : public Layer * @note Use this constructor for ELU activation function. * @param alpha Scale parameter for the negative factor. */ - ELUType(const double alpha); + ELU(const double alpha); - //! Clone the ELUType object. This handles polymorphism correctly. - ELUType* Clone() const { return new ELUType(*this); } /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. @@ -134,6 +135,7 @@ class ELUType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -145,13 +147,29 @@ class ELUType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const DataType& input, const DataType& gy, DataType& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the non zero gradient. double const& Alpha() const { return alpha; } //! Modify the non zero gradient. double& Alpha() { return alpha; } + //! Get the value of deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of deterministic parameter. + bool& Deterministic() { return deterministic; } + //! Get the lambda parameter. double const& Lambda() const { return lambda; } @@ -162,27 +180,31 @@ class ELUType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Locally stored first derivative of the activation function. - OutputType derivative; + arma::mat derivative; //! ELU Hyperparameter (0 < alpha) //! SELU parameter fixed to 1.6732632423543774 for normalized inputs. double alpha; - //! Lambda parameter used for multiplication of ELU function. + //! Lambda Parameter used for multiplication of ELU function. //! For ELU activation function, lambda = 1. //! For SELU activation function, lambda = 1.0507009873554802 for normalized //! inputs. double lambda; -}; // class ELUType -// Convenience typedefs. + //! If true the derivative computation is disabled, see notes above. + bool deterministic; +}; // class ELU -// Standard flexible ReLU layer. -typedef ELUType ELU; - -// Standard ELU layer. -typedef ELUType SELU; +// Template alias for SELU using ELU class. +using SELU = ELU; } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/elu_impl.hpp b/src/mlpack/methods/ann/layer/elu_impl.hpp similarity index 53% rename from src/mlpack/methods/ann/layer/not_adapted/elu_impl.hpp rename to src/mlpack/methods/ann/layer/elu_impl.hpp index b81d2caf89..9615604c86 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/elu_impl.hpp +++ b/src/mlpack/methods/ann/layer/elu_impl.hpp @@ -26,59 +26,66 @@ namespace ann /** Artificial Neural Network. */ { // This constructor is called for SELU activation function. The values of // alpha and lambda are constant for normalized inputs. -template -ELUType::ELUType() : +template +ELU::ELU() : alpha(1.6732632423543774), - lambda(1.0507009873554802) + lambda(1.0507009873554802), + deterministic(false) { // Nothing to do here. } -// This constructor is called for ELU activation function. The value of lambda -// is fixed and equal to 1. 'alpha' is a hyperparameter. -template -ELUType::ELUType(const double alpha) : +// This constructor is called for ELU activation function. The value of lambda +// is fixed and equal to 1. 'alpha' is a hyperparameter. +template +ELU::ELU(const double alpha) : alpha(alpha), - lambda(1) + lambda(1), + deterministic(false) { // Nothing to do here. } +template template -void ELUType::Forward( +void ELU::Forward( const InputType& input, OutputType& output) { - output.ones(); + output = arma::ones(arma::size(input)); for (size_t i = 0; i < input.n_elem; ++i) { if (input(i) < DBL_MAX) { - output(i) = (input(i) > 0) ? lambda * input(i) : lambda * alpha * - (std::exp(input(i)) - 1); + output(i) = (input(i) > 0) ? lambda * input(i) : lambda * + alpha * (std::exp(input(i)) - 1); } } - if (!deterministic) - { - for (size_t i = 0; i < input.n_elem; ++i) - derivative(i) = (input(i) > 0) ? lambda : output(i) + lambda * alpha; - } + if (!deterministic) + { + derivative.set_size(arma::size(input)); + for (size_t i = 0; i < input.n_elem; ++i) + { + derivative(i) = (input(i) > 0) ? lambda : output(i) + + lambda * alpha; + } + } } -template -void ELUType::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) +template +template +void ELU::Backward( + const DataType& /* input */, const DataType& gy, DataType& g) { g = gy % derivative; } -template +template template -void ELUType::serialize( - Archive& ar, const uint32_t /* version */) +void ELU::serialize( + Archive& ar, + const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(alpha)); ar(CEREAL_NVP(lambda)); } diff --git a/src/mlpack/methods/ann/layer/not_adapted/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp similarity index 72% rename from src/mlpack/methods/ann/layer/not_adapted/fast_lstm.hpp rename to src/mlpack/methods/ann/layer/fast_lstm.hpp index 120442b145..80ddcabca6 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/fast_lstm.hpp * @author Marcus Edel @@ -16,7 +15,6 @@ #include #include -#include "layer.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -56,36 +54,36 @@ namespace ann /** Artificial Neural Network. */ { * * \see LSTM for a standard implementation of the LSTM layer. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class FastLSTMType : public Layer +class FastLSTM { public: // Convenience typedefs. - typedef typename InputType::elem_type InputET; - typedef typename OutputType::elem_type OutputET; + typedef typename InputDataType::elem_type InputElemType; + typedef typename OutputDataType::elem_type ElemType; - //! Create the FastLSTMType object. - FastLSTMType(); + //! Create the Fast LSTM object. + FastLSTM(); //! Copy Constructor - FastLSTMType(const FastLSTMType& layer); + FastLSTM(const FastLSTM& layer); //! Move Constructor - FastLSTMType(FastLSTMType&& layer); + FastLSTM(FastLSTM&& layer); //! Copy assignment operator - FastLSTMType& operator=(const FastLSTMType& layer); + FastLSTM& operator=(const FastLSTM& layer); //! Move assignment operator - FastLSTMType& operator=(FastLSTMType&& layer); + FastLSTM& operator=(FastLSTM&& layer); /** * Create the Fast LSTM layer object using the specified parameters. @@ -94,12 +92,9 @@ class FastLSTMType : public Layer * @param outSize The number of output units. * @param rho Maximum number of steps to backpropagate through time (BPTT). */ - FastLSTMType(const size_t inSize, - const size_t outSize, - const size_t rho = std::numeric_limits::max()); - - //! Clone the FastLSTMType object. This handles polymorphism correctly. - FastLSTMType* Clone() const { return new FastLSTMType(*this); } + FastLSTM(const size_t inSize, + const size_t outSize, + const size_t rho = std::numeric_limits::max()); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -108,6 +103,7 @@ class FastLSTMType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -119,16 +115,17 @@ class FastLSTMType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ + template void Backward(const InputType& input, - const OutputType& gy, - OutputType& g); + const ErrorType& gy, + GradientType& g); - /** + /* * Reset the layer parameter. */ void Reset(); - /** + /* * Resets the cell to accept a new input. This breaks the BPTT chain starts a * new one. * @@ -136,16 +133,17 @@ class FastLSTMType : public Layer */ void ResetCell(const size_t size); - /** + /* * Calculate the gradient using the output delta and the input activation. * * @param input The input parameter used for calculating the gradient. * @param error The calculated error. * @param gradient The calculated gradient. */ + template void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + const ErrorType& error, + GradientType& gradient); //! Get the maximum number of steps to backpropagate through time (BPTT). size_t Rho() const { return rho; } @@ -153,9 +151,24 @@ class FastLSTMType : public Layer size_t& Rho() { return rho; } //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return grad; } + //! Modify the gradient. + OutputDataType& Gradient() { return grad; } //! Get the number of input units. size_t InSize() const { return inSize; } @@ -169,15 +182,14 @@ class FastLSTMType : public Layer return 4 * outSize * inSize + 4 * outSize + 4 * outSize * outSize; } - const std::vector OutputDimensions() const + //! Get the shape of the input. + size_t InputShape() const { - std::vector result(inputDimensions.size(), 0); - result[0] = outSize; - return result; + return inSize; } /** - * Serialize the layer. + * Serialize the layer */ template void serialize(Archive& ar, const uint32_t /* version */); @@ -189,6 +201,7 @@ class FastLSTMType : public Layer * @param input The input data. * @param sigmoid The matrix to store the sigmoid approximation into. */ + template void FastSigmoid(const InputType& input, OutputType& sigmoids) { for (size_t i = 0; i < input.n_elem; ++i) @@ -201,10 +214,10 @@ class FastLSTMType : public Layer * @param data The given data sample for the sigmoid approximation. * @tparam The sigmoid approximation. */ - OutputET FastSigmoid(const InputET data) + ElemType FastSigmoid(const InputElemType data) { - OutputET x = 0.5 * data; - OutputET z; + ElemType x = 0.5 * data; + ElemType z; if (x >= 0) { if (x < 1.7) @@ -216,7 +229,7 @@ class FastLSTMType : public Layer } else { - OutputET xx = -x; + ElemType xx = -x; if (xx < 1.7) z = -(1.5 * xx / (1 + xx)); else if (xx < 3) @@ -247,10 +260,10 @@ class FastLSTMType : public Layer size_t gradientStep; //! Locally-stored weight object. - OutputType weights; + OutputDataType weights; //! Locally-stored previous output. - OutputType prevOutput; + OutputDataType prevOutput; //! Locally-stored batch size. size_t batchSize; @@ -263,50 +276,56 @@ class FastLSTMType : public Layer size_t gradientStepIdx; //! Locally-stored cell activation error. - OutputType cellActivationError; + OutputDataType cellActivationError; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType grad; //! Locally-stored output parameter object. - OutputType outputParameter; + OutputDataType outputParameter; //! Weights between the output and gate. - OutputType output2GateWeight; + OutputDataType output2GateWeight; //! Weights between the input and gate. - OutputType input2GateWeight; + OutputDataType input2GateWeight; //! Bias between the input and gate. - OutputType input2GateBias; + OutputDataType input2GateBias; //! Locally-stored gate parameter. - OutputType gate; + OutputDataType gate; //! Locally-stored gate activation. - OutputType gateActivation; + OutputDataType gateActivation; //! Locally-stored state activation. - OutputType stateActivation; + OutputDataType stateActivation; //! Locally-stored cell parameter. - OutputType cell; + OutputDataType cell; //! Locally-stored cell activation error. - OutputType cellActivation; + OutputDataType cellActivation; //! Locally-stored foget gate error. - OutputType forgetGateError; + OutputDataType forgetGateError; //! Locally-stored previous error. - OutputType prevError; + OutputDataType prevError; + + //! Locally-stored output parameters. + OutputDataType outParameter; //! Locally-stored current rho size. size_t rhoSize; //! Current backpropagate through time steps. size_t bpttSteps; -}; // class FastLSTMType. - -// Standard FastLSTM layer. -typedef FastLSTMType FastLSTM; +}; // class FastLSTM } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp similarity index 81% rename from src/mlpack/methods/ann/layer/not_adapted/fast_lstm_impl.hpp rename to src/mlpack/methods/ann/layer/fast_lstm_impl.hpp index 46b6824c0d..c72416bdeb 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/fast_lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp @@ -19,14 +19,14 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -FastLSTMType::FastLSTMType() +template +FastLSTM::FastLSTM() { // Nothing to do here. } -template -FastLSTMType::FastLSTMType( +template +FastLSTM::FastLSTM( const size_t inSize, const size_t outSize, const size_t rho) : inSize(inSize), outSize(outSize), @@ -45,8 +45,8 @@ FastLSTMType::FastLSTMType( weights.set_size(WeightSize(), 1); } -template -FastLSTMType::FastLSTMType(const FastLSTMType& layer) : +template +FastLSTM::FastLSTM(const FastLSTM& layer) : inSize(layer.inSize), outSize(layer.outSize), rho(layer.rho), @@ -57,14 +57,15 @@ FastLSTMType::FastLSTMType(const FastLSTMType& layer) : batchSize(layer.batchSize), batchStep(layer.batchStep), gradientStepIdx(layer.gradientStepIdx), + grad(layer.grad), rhoSize(layer.rho), bpttSteps(layer.bpttSteps) { // Nothing to do here. } -template -FastLSTMType::FastLSTMType(FastLSTMType&& layer) : +template +FastLSTM::FastLSTM(FastLSTM&& layer) : inSize(std::move(layer.inSize)), outSize(std::move(layer.outSize)), rho(std::move(layer.rho)), @@ -75,15 +76,16 @@ FastLSTMType::FastLSTMType(FastLSTMType&& layer) : batchSize(std::move(layer.batchSize)), batchStep(std::move(layer.batchStep)), gradientStepIdx(std::move(layer.gradientStepIdx)), + grad(std::move(layer.grad)), rhoSize(std::move(layer.rho)), bpttSteps(std::move(layer.bpttSteps)) { // Nothing to do here. } -template -FastLSTMType& -FastLSTMType::operator=(const FastLSTMType& layer) +template +FastLSTM& +FastLSTM::operator=(const FastLSTM& layer) { if (this != &layer) { @@ -97,15 +99,16 @@ FastLSTMType::operator=(const FastLSTMType& layer) batchSize = layer.batchSize; batchStep = layer.batchStep; gradientStepIdx = layer.gradientStepIdx; + grad = layer.grad; rhoSize = layer.rho; bpttSteps = layer.bpttSteps; } return *this; } -template -FastLSTMType& -FastLSTMType::operator=(FastLSTMType&& layer) +template +FastLSTM& +FastLSTM::operator=(FastLSTM&& layer) { if (this != &layer) { @@ -119,30 +122,31 @@ FastLSTMType::operator=(FastLSTMType&& layer) batchSize = std::move(layer.batchSize); batchStep = std::move(layer.batchStep); gradientStepIdx = std::move(layer.gradientStepIdx); + grad = std::move(layer.grad); rhoSize = std::move(layer.rho); bpttSteps = std::move(layer.bpttSteps); } return *this; } -template -void FastLSTMType::Reset() +template +void FastLSTM::Reset() { // Set the weight parameter for the input to gate layer (linear layer) using // the overall layer parameter matrix. - input2GateWeight = OutputType(weights.memptr(), + input2GateWeight = OutputDataType(weights.memptr(), 4 * outSize, inSize, false, false); - input2GateBias = OutputType(weights.memptr() + input2GateWeight.n_elem, + input2GateBias = OutputDataType(weights.memptr() + input2GateWeight.n_elem, 4 * outSize, 1, false, false); // Set the weight parameter for the output to gate layer // (linear no bias layer) using the overall layer parameter matrix. - output2GateWeight = OutputType(weights.memptr() + input2GateWeight.n_elem + output2GateWeight = OutputDataType(weights.memptr() + input2GateWeight.n_elem + input2GateBias.n_elem, 4 * outSize, outSize, false, false); } -template -void FastLSTMType::ResetCell(const size_t size) +template +void FastLSTM::ResetCell(const size_t size) { if (size == std::numeric_limits::max()) return; @@ -175,8 +179,9 @@ void FastLSTMType::ResetCell(const size_t size) outParameter.zeros(outSize, (size + 1) * batchSize); } +template template -void FastLSTMType::Forward( +void FastLSTM::Forward( const InputType& input, OutputType& output) { // Check if the batch size changed, the number of cols is defines the input @@ -193,8 +198,8 @@ void FastLSTMType::Forward( forwardStep, forwardStep + batchStep); gate.cols(forwardStep, forwardStep + batchStep).each_col() += input2GateBias; - InputType sigmoidOut(gateActivation.colptr(forwardStep), - gateActivation.n_rows, batchStep, false, false); + arma::subview sigmoidOut = gateActivation.cols(forwardStep, + forwardStep + batchStep); FastSigmoid( gate.submat(0, forwardStep, 3 * outSize - 1, forwardStep + batchStep), sigmoidOut); @@ -242,19 +247,20 @@ void FastLSTMType::Forward( } } -template -void FastLSTMType::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) +template +template +void FastLSTM::Backward( + const InputType& /* input */, const ErrorType& gy, GradientType& g) { - OutputType gyLocal; + ErrorType gyLocal; if (gradientStepIdx > 0) { gyLocal = gy + output2GateWeight.t() * prevError; } else { - gyLocal = OutputType(((OutputType&) gy).memptr(), gy.n_rows, gy.n_cols, - false, false); + gyLocal = ErrorType(((ErrorType&) gy).memptr(), gy.n_rows, gy.n_cols, false, + false); } cellActivationError = gyLocal % gateActivation.submat(outSize, @@ -313,11 +319,12 @@ void FastLSTMType::Backward( } } -template -void FastLSTMType::Gradient( +template +template +void FastLSTM::Gradient( const InputType& input, - const OutputType& /* error */, - OutputType& gradient) + const ErrorType& /* error */, + GradientType& gradient) { // Gradient of the input to gate layer. gradient.submat(0, 0, input2GateWeight.n_elem - 1, 0) = @@ -341,13 +348,11 @@ void FastLSTMType::Gradient( } } -template +template template -void FastLSTMType::serialize( +void FastLSTM::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(weights)); ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(outSize)); @@ -366,10 +371,7 @@ void FastLSTMType::serialize( ar(CEREAL_NVP(cellActivation)); ar(CEREAL_NVP(forgetGateError)); ar(CEREAL_NVP(prevError)); - - // Restore aliases. - if (Archive::is_loading::value) - Reset(); + ar(CEREAL_NVP(outParameter)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/flatten_t_swish.hpp b/src/mlpack/methods/ann/layer/flatten_t_swish.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/flatten_t_swish.hpp rename to src/mlpack/methods/ann/layer/flatten_t_swish.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/flatten_t_swish_impl.hpp b/src/mlpack/methods/ann/layer/flatten_t_swish_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/flatten_t_swish_impl.hpp rename to src/mlpack/methods/ann/layer/flatten_t_swish_impl.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp similarity index 50% rename from src/mlpack/methods/ann/layer/not_adapted/flexible_relu.hpp rename to src/mlpack/methods/ann/layer/flexible_relu.hpp index 279d00a5f7..7c2ea9e4dc 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/flexible_relu.hpp +++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp @@ -3,9 +3,10 @@ * @author Aarush Gupta * @author Manthan-R-Sheth * - * Definition of the FlexibleReLU layer as described by Suo Qiu, Xiangmin Xu and - * Bolun Cai in "FReLU: Flexible Rectified Linear Units for Improving - * Convolutional Neural Networks". + * Definition of FlexibleReLU layer as described by + * Suo Qiu, Xiangmin Xu and Bolun Cai in + * "FReLU: Flexible Rectified Linear Units for Improving Convolutional + * Neural Networks", 2018 * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -17,8 +18,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /**Artificial Neural Network*/ { @@ -28,10 +27,10 @@ namespace ann /**Artificial Neural Network*/ { * @f{eqnarray*}{ * f(x) &=& \max(0,x)+alpha \\ * f'(x) &=& \left\{ - * \begin{array}{lr} - * 1 & : x > 0 \\ - * 0 & : x \le 0 - * \end{array} + * \begin{array}{lr} + * 1 & : x > 0 \\ + * 0 & : x \le 0 + * \end{array} * \right. * @f} * @@ -48,33 +47,34 @@ namespace ann /**Artificial Neural Network*/ { * } * @endcode * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mar, + * arma::sp_mat or arma::cube) + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube) */ -template -class FlexibleReLUType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class FlexibleReLU { public: /** - * Create the FlexibleReLU object using the specified alpha parameter. - * The trainable alpha parameter controls the range of the ReLU function. - * (Default alpha = 0). * - * @param alpha Parameter to adjust the range of the ReLU function. + * Create the FlexibleReLU object using the specified parameters. + * The non zero parameter can be adjusted by specifying the parameter + * alpha which controls the range of the relu function. (Default alpha = 0) + * This parameter is trainable. + * + * @param alpha Parameter for adjusting the range of the relu function. + * */ - FlexibleReLUType(const double alpha = 0); - - //! Clone the FlexibleReLUType object. This handles polymorphism correctly. - FlexibleReLUType* Clone() const { return new FlexibleReLUType(*this); } + FlexibleReLU(const double alpha = 0); /** - * Reset the layer parameter (alpha). The method is called to - * assign the allocated memory to the learnable layer parameter. + * Reset the layer parameter. */ - void SetWeights(typename OutputType::elem_type* weightsPtr); + void Reset(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -83,6 +83,7 @@ class FlexibleReLUType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -94,7 +95,8 @@ class FlexibleReLUType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const DataType& input, const DataType& gy, DataType& g); /** * Calculate the gradient using the output delta and the input activation. @@ -103,22 +105,36 @@ class FlexibleReLUType : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - OutputType const& Parameters() const { return alpha; } + OutputDataType const& Parameters() const { return alpha; } //! Modify the parameters. - OutputType& Parameters() { return alpha; } + OutputDataType& Parameters() { return alpha; } - //! Get the parameter controlling the range of the ReLU function. - const double& Alpha() const { return alpha; } - //! Modify the parameter controlling the range of the ReLU function. + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta;} + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the parameter controlling the range of the relu function. + double const& Alpha() const { return alpha; } + //! Modify the parameter controlling the range of the relu function. double& Alpha() { return alpha; } - const size_t WeightSize() const { return 1; } - /** * Serialize the layer. */ @@ -126,20 +142,21 @@ class FlexibleReLUType : public Layer void serialize(Archive& ar, const uint32_t /* version*/); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Parameter object. - OutputType alpha; + OutputDataType alpha; - //! Parameter controlling the range of the ReLU function. + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Parameter controlling the range of the rectifier function double userAlpha; - - //! Whether or not a forward pass has ever been performed. - bool initialized; -}; // class FlexibleReLUType - -// Convenience typedefs. - -// Standard flexible ReLU layer. -typedef FlexibleReLUType FlexibleReLU; +}; // class FlexibleReLU } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/flexible_relu_impl.hpp b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp similarity index 51% rename from src/mlpack/methods/ann/layer/not_adapted/flexible_relu_impl.hpp rename to src/mlpack/methods/ann/layer/flexible_relu_impl.hpp index 74af9b611e..ffd695b3e5 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/flexible_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/flexible_relu_impl.hpp @@ -18,67 +18,66 @@ #define MLPACK_METHODS_ANN_LAYER_FLEXIBLERELU_IMPL_HPP #include "flexible_relu.hpp" +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -FlexibleReLUType::FlexibleReLUType(const double alpha) : - userAlpha(alpha), - initialized(false) +template +FlexibleReLU::FlexibleReLU( + const double alpha) : userAlpha(alpha) { this->alpha.set_size(1, 1); - this->alpha(0) = alpha; + this->alpha(0) = userAlpha; } -template -void FlexibleReLUType::SetWeights( - typename OutputType::elem_type* weightsPtr) +template +void FlexibleReLU::Reset() { - alpha = OutputType(weightsPtr, 1, 1, false, false); + //! Set value of alpha to the one given by user. + alpha(0) = userAlpha; } +template template -void FlexibleReLUType::Forward( +void FlexibleReLU::Forward( const InputType& input, OutputType& output) { - if (!initialized) - { - alpha[0] = userAlpha; - initialized = true; - } - output = arma::clamp(input, 0.0, DBL_MAX) + alpha(0); } -template -void FlexibleReLUType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) +template +template +void FlexibleReLU::Backward( + const DataType& input, const DataType& gy, DataType& g) { - // Compute the first derivative of FlexibleReLU function. + //! Compute the first derivative of FlexibleReLU function. g = gy % arma::clamp(arma::sign(input), 0.0, 1.0); } -template -void FlexibleReLUType::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) +template +template +void FlexibleReLU::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { + if (gradient.n_elem == 0) + { + gradient.set_size(1, 1); + } + gradient(0) = arma::accu(error) / input.n_cols; } -template + +template template -void FlexibleReLUType::serialize( +void FlexibleReLU::serialize( Archive& ar, const uint32_t /* version*/) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(alpha)); - ar(CEREAL_NVP(userAlpha)); - ar(CEREAL_NVP(initialized)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/glimpse.hpp b/src/mlpack/methods/ann/layer/glimpse.hpp similarity index 80% rename from src/mlpack/methods/ann/layer/not_adapted/glimpse.hpp rename to src/mlpack/methods/ann/layer/glimpse.hpp index ee25a0cbb6..99a268b6a8 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/glimpse.hpp +++ b/src/mlpack/methods/ann/layer/glimpse.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/glimpse.hpp * @author Marcus Edel @@ -77,16 +76,16 @@ class MeanPoolingRule * (down-scaled cropped images) of increasing scale around a given location in a * given image. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class GlimpseType : public Layer +class Glimpse { public: /** @@ -101,12 +100,12 @@ class GlimpseType : public Layer * @param inputWidth The input width of the given input data. * @param inputHeight The input height of the given input data. */ - GlimpseType(const size_t inSize = 0, - const size_t size = 0, - const size_t depth = 3, - const size_t scale = 2, - const size_t inputWidth = 0, - const size_t inputHeight = 0); + Glimpse(const size_t inSize = 0, + const size_t size = 0, + const size_t depth = 3, + const size_t scale = 2, + const size_t inputWidth = 0, + const size_t inputHeight = 0); /** * Ordinary feed forward pass of the glimpse layer. @@ -114,7 +113,8 @@ class GlimpseType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the glimpse layer. @@ -123,13 +123,27 @@ class GlimpseType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType& OutputParameter() const {return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the detla. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Set the locationthe x and y coordinate of the center of the output //! glimpse. - void Location(const arma::mat& location) { this->location = location; } + void Location(const arma::mat& location) + { + this->location = location; + } //! Get the input width. size_t const& InputWidth() const { return inputWidth; } @@ -151,6 +165,11 @@ class GlimpseType : public Layer //! Modify the output height. size_t& OutputHeight() { return outputHeight; } + //! Get the value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } + //! Get the number of patches to crop per glimpse. size_t const& Depth() const { return depth; } @@ -163,14 +182,10 @@ class GlimpseType : public Layer //! Get the used glimpse size (height = width). size_t GlimpseSize() const { return size;} - const std::vector OutputDimensions() const + //! Get the shape of the input. + size_t InputShape() const { - std::vector result(inputDimensions.size(), 0); - result[0] = outputWidth; - result[1] = outputHeight; - for (size_t i = 2; i < inputDimensions.size(); ++i) - result[i] = inputDimensions[i]; - return result; + return inSize; } /** @@ -180,7 +195,7 @@ class GlimpseType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: - /** + /* * Transform the given input by changing rows to columns. * * @param w The input matrix used to perform the transformation. @@ -220,9 +235,10 @@ class GlimpseType : public Layer * @param input The input to be apply the pooling rule. * @param output The pooled result. */ + template void Pooling(const size_t kSize, - const InputType& input, - OutputType& output) + const arma::Mat& input, + arma::Mat& output) { const size_t rStep = kSize; const size_t cStep = kSize; @@ -244,20 +260,21 @@ class GlimpseType : public Layer * @param error The error used to perform the unpooling operation. * @param output The pooled result. */ - void Unpooling(const InputType& input, - const OutputType& error, - OutputType& output) + template + void Unpooling(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& output) { const size_t rStep = input.n_rows / error.n_rows; const size_t cStep = input.n_cols / error.n_cols; - OutputType unpooledError; + arma::Mat unpooledError; for (size_t j = 0; j < input.n_cols; j += cStep) { for (size_t i = 0; i < input.n_rows; i += rStep) { - const InputType& inputArea = input(arma::span(i, i + rStep - 1), - arma::span(j, j + cStep - 1)); + const arma::Mat& inputArea = input(arma::span(i, i + rStep - 1), + arma::span(j, j + cStep - 1)); pooling.Unpooling(inputArea, error(i / rStep, j / cStep), unpooledError); @@ -275,7 +292,8 @@ class GlimpseType : public Layer * @param input The input to be apply the ReSampling rule. * @param output The pooled result. */ - void ReSampling(const InputType& input, OutputType& output) + template + void ReSampling(const arma::Mat& input, arma::Mat& output) { double wRatio = (double) (input.n_rows - 1) / (size - 1); double hRatio = (double) (input.n_cols - 1) / (size - 1); @@ -319,9 +337,10 @@ class GlimpseType : public Layer * @param error The error used to perform the DownwardReSampling operation. * @param output The DownwardReSampled result. */ - void DownwardReSampling(const InputType& input, - const OutputType& error, - OutputType& output) + template + void DownwardReSampling(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& output) { double iWidth = input.n_rows - 1; double iHeight = input.n_cols - 1; @@ -385,30 +404,36 @@ class GlimpseType : public Layer //! Locally-stored output height. size_t outputHeight; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Locally-stored depth of the input. size_t inputDepth; //! Locally-stored transformed input parameter. - arma::Cube inputTemp; + arma::cube inputTemp; //! Locally-stored transformed output parameter. - arma::Cube outputTemp; + arma::cube outputTemp; //! The x and y coordinate of the center of the output glimpse. - OutputType location; + arma::mat location; //! Locally-stored object to perform the mean pooling operation. MeanPoolingRule pooling; //! Location-stored module location parameter. - std::vector locationParameter; + std::vector locationParameter; //! Location-stored transformed gradient paramter. - arma::Cube gTemp; -}; // class GlimpseType + arma::cube gTemp; -// Standard Glimpse layer. -typedef GlimpseType Glimpse; + //! If true use maximum a posteriori during the forward pass. + bool deterministic; +}; // class GlimpseLayer } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/glimpse_impl.hpp b/src/mlpack/methods/ann/layer/glimpse_impl.hpp similarity index 76% rename from src/mlpack/methods/ann/layer/not_adapted/glimpse_impl.hpp rename to src/mlpack/methods/ann/layer/glimpse_impl.hpp index a76b77f70c..b66379bfd1 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/glimpse_impl.hpp +++ b/src/mlpack/methods/ann/layer/glimpse_impl.hpp @@ -20,8 +20,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -GlimpseType::GlimpseType( +template +Glimpse::Glimpse( const size_t inSize, const size_t size, const size_t depth, @@ -42,14 +42,13 @@ GlimpseType::GlimpseType( // Nothing to do here. } -template -void GlimpseType::Forward( - const InputType& input, OutputType& output) +template +template +void Glimpse::Forward( + const arma::Mat& input, arma::Mat& output) { - inputTemp = arma::Cube(input.colptr(0), - inputWidth, inputHeight, inSize); - outputTemp = arma::Cube(size, size, depth * - inputTemp.n_slices); + inputTemp = arma::cube(input.colptr(0), inputWidth, inputHeight, inSize); + outputTemp = arma::Cube(size, size, depth * inputTemp.n_slices); location = input.submat(0, 1, 1, 1); @@ -67,8 +66,7 @@ void GlimpseType::Forward( { size_t padSize = std::floor((glimpseSize - 1) / 2); - arma::Cube inputPadded = - arma::zeros>( + arma::Cube inputPadded = arma::zeros >( inputTemp.n_rows + padSize * 2, inputTemp.n_cols + padSize * 2, inputTemp.n_slices / inSize); @@ -100,8 +98,9 @@ void GlimpseType::Forward( for (size_t j = (inputIdx + depthIdx * (depth - 1)), paddedSlice = 0; j < outputTemp.n_slices; j += (inSize * depth), paddedSlice++) { - InputType poolingInput = inputPadded.subcube(x, y, paddedSlice, - x + glimpseSize - 1, y + glimpseSize - 1, paddedSlice); + arma::Mat poolingInput = inputPadded.subcube(x, y, + paddedSlice, x + glimpseSize - 1, y + glimpseSize - 1, + paddedSlice); if (scale == 2) { @@ -121,19 +120,19 @@ void GlimpseType::Forward( outputTemp.slice(i) = arma::trans(outputTemp.slice(i)); } - output = OutputType(outputTemp.memptr(), outputTemp.n_elem, 1); + output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem, 1); outputWidth = outputTemp.n_rows; outputHeight = outputTemp.n_cols; } -template -void GlimpseType::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) +template +template +void Glimpse::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { // Generate a cube using the backpropagated error matrix. - arma::Cube mappedError = - arma::zeros>(outputWidth, + arma::Cube mappedError = arma::zeros(outputWidth, outputHeight, 1); location = locationParameter.back(); @@ -143,13 +142,13 @@ void GlimpseType::Backward( { for (size_t i = 0; i < gy.n_cols; ++i) { - mappedError.slice(s + i) = OutputType(gy.memptr(), + mappedError.slice(s + i) = arma::Mat(gy.memptr(), outputWidth, outputHeight); } } - gTemp = arma::zeros>( - inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices); + gTemp = arma::zeros(inputTemp.n_rows, inputTemp.n_cols, + inputTemp.n_slices); for (size_t inputIdx = 0; inputIdx < inSize; inputIdx++) { @@ -158,8 +157,7 @@ void GlimpseType::Backward( { size_t padSize = std::floor((glimpseSize - 1) / 2); - arma::Cube inputPadded = - arma::zeros>( + arma::Cube inputPadded = arma::zeros >( inputTemp.n_rows + padSize * 2, inputTemp.n_cols + padSize * 2, inputTemp.n_slices / inSize); @@ -186,8 +184,9 @@ void GlimpseType::Backward( for (size_t j = (inputIdx + depthIdx * (depth - 1)), paddedSlice = 0; j < mappedError.n_slices; j += (inSize * depth), paddedSlice++) { - OutputType poolingOutput = inputPadded.subcube(x, y, paddedSlice, - x + glimpseSize - 1, y + glimpseSize - 1, paddedSlice); + arma::Mat poolingOutput = inputPadded.subcube(x, y, + paddedSlice, x + glimpseSize - 1, y + glimpseSize - 1, + paddedSlice); if (scale == 2) { @@ -212,16 +211,14 @@ void GlimpseType::Backward( } Transform(gTemp); - g = OutputType(gTemp.memptr(), gTemp.n_elem, 1); + g = arma::mat(gTemp.memptr(), gTemp.n_elem, 1); } -template +template template -void GlimpseType::serialize( +void Glimpse::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(size)); ar(CEREAL_NVP(depth)); diff --git a/src/mlpack/methods/ann/layer/not_adapted/group_norm.hpp b/src/mlpack/methods/ann/layer/group_norm.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/group_norm.hpp rename to src/mlpack/methods/ann/layer/group_norm.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/group_norm_impl.hpp b/src/mlpack/methods/ann/layer/group_norm_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/group_norm_impl.hpp rename to src/mlpack/methods/ann/layer/group_norm_impl.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/gru.hpp b/src/mlpack/methods/ann/layer/gru.hpp similarity index 67% rename from src/mlpack/methods/ann/layer/not_adapted/gru.hpp rename to src/mlpack/methods/ann/layer/gru.hpp index 290adb0276..c895ee7f81 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/gru.hpp +++ b/src/mlpack/methods/ann/layer/gru.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/gru.hpp * @author Sumedh Ghaisas @@ -32,6 +31,9 @@ #include +#include "../visitor/delta_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" + #include "layer_types.hpp" #include "add_merge.hpp" #include "sequential.hpp" @@ -44,16 +46,16 @@ namespace ann /** Artificial Neural Network. */ { * * This cell can be used in RNN networks. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class GRU : public Layer +class GRU { public: //! Create the GRU object. @@ -77,7 +79,8 @@ class GRU : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -88,9 +91,10 @@ class GRU : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -99,9 +103,10 @@ class GRU : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& /* error */, - OutputType& /* gradient */); + template + void Gradient(const arma::Mat& input, + const arma::Mat& /* error */, + arma::Mat& /* gradient */); /* * Resets the cell to accept a new input. This breaks the BPTT chain starts a @@ -111,18 +116,38 @@ class GRU : public Layer */ void ResetCell(const size_t size); + //! The value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } + //! Get the maximum number of steps to backpropagate through time (BPTT). size_t Rho() const { return rho; } //! Modify the maximum number of steps to backpropagate through time (BPTT). size_t& Rho() { return rho; } //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the model modules. - std::vector*>& Model() { return network; } + std::vector >& Model() { return network; } //! Get the number of input units. size_t InSize() const { return inSize; } @@ -156,28 +181,37 @@ class GRU : public Layer size_t batchSize; //! Locally-stored weight object. - OutputType weights; + OutputDataType weights; //! Locally-stored input 2 gate module. - Layer* input2GateModule; + LayerTypes<> input2GateModule; //! Locally-stored output 2 gate module. - Layer* output2GateModule; + LayerTypes<> output2GateModule; //! Locally-stored output hidden state 2 gate module. - Layer* outputHidden2GateModule; + LayerTypes<> outputHidden2GateModule; //! Locally-stored input gate module. - Layer* inputGateModule; + LayerTypes<> inputGateModule; //! Locally-stored hidden state module. - Layer* hiddenStateModule; + LayerTypes<> hiddenStateModule; //! Locally-stored forget gate module. - Layer* forgetGateModule; + LayerTypes<> forgetGateModule; + + //! Locally-stored output parameter visitor. + OutputParameterVisitor outputParameterVisitor; + + //! Locally-stored delta visitor. + DeltaVisitor deltaVisitor; + + //! Locally-stored delete visitor. + DeleteVisitor deleteVisitor; //! Locally-stored list of network modules. - std::vector*> network; + std::vector > network; //! Locally-stored number of forward steps. size_t forwardStep; @@ -189,34 +223,34 @@ class GRU : public Layer size_t gradientStep; //! Locally-stored output parameters. - std::list outParameter; + std::list outParameter; //! Matrix of all zeroes to initialize the output - OutputType allZeros; + arma::mat allZeros; //! Iterator pointed to the last output produced by the cell - typename std::list::iterator prevOutput; + std::list::iterator prevOutput; //! Iterator pointed to the last output processed by backward - typename std::list::iterator backIterator; + std::list::iterator backIterator; //! Iterator pointed to the last output processed by gradient - typename std::list::iterator gradIterator; + std::list::iterator gradIterator; //! Locally-stored previous error. - OutputType prevError; + arma::mat prevError; //! If true dropout and scaling is disabled, see notes above. bool deterministic; //! Locally-stored delta object. - OutputType delta; + OutputDataType delta; //! Locally-stored gradient object. - OutputType gradient; + OutputDataType gradient; //! Locally-stored output parameter object. - OutputType outputParameter; + OutputDataType outputParameter; }; // class GRU } // namespace ann diff --git a/src/mlpack/methods/ann/layer/gru_impl.hpp b/src/mlpack/methods/ann/layer/gru_impl.hpp new file mode 100644 index 0000000000..e9b71ebb97 --- /dev/null +++ b/src/mlpack/methods/ann/layer/gru_impl.hpp @@ -0,0 +1,411 @@ +/** + * @file methods/ann/layer/gru_impl.hpp + * @author Sumedh Ghaisas + * + * Implementation of the GRU class, which implements a gru network + * layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_GRU_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_GRU_IMPL_HPP + +// In case it hasn't yet been included. +#include "gru.hpp" + +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +GRU::GRU() +{ + // Nothing to do here. +} + +template +GRU::GRU( + const size_t inSize, + const size_t outSize, + const size_t rho) : + inSize(inSize), + outSize(outSize), + rho(rho), + batchSize(1), + forwardStep(0), + backwardStep(0), + gradientStep(0), + deterministic(false) +{ + // Input specific linear layers(for zt, rt, ot). + input2GateModule = new Linear<>(inSize, 3 * outSize); + + // Previous output gates (for zt and rt). + output2GateModule = new LinearNoBias<>(outSize, 2 * outSize); + + // Previous output gate for ot. + outputHidden2GateModule = new LinearNoBias<>(outSize, outSize); + + network.push_back(input2GateModule); + network.push_back(output2GateModule); + network.push_back(outputHidden2GateModule); + + inputGateModule = new SigmoidLayer<>(); + forgetGateModule = new SigmoidLayer<>(); + hiddenStateModule = new TanHLayer<>(); + + network.push_back(inputGateModule); + network.push_back(hiddenStateModule); + network.push_back(forgetGateModule); + + prevError = arma::zeros(3 * outSize, batchSize); + + allZeros = arma::zeros(outSize, batchSize); + + outParameter.emplace_back(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true); + + prevOutput = outParameter.begin(); + backIterator = outParameter.end(); + gradIterator = outParameter.end(); +} + +template +template +void GRU::Forward( + const arma::Mat& input, arma::Mat& output) +{ + if (input.n_cols != batchSize) + { + batchSize = input.n_cols; + prevError.resize(3 * outSize, batchSize); + allZeros.zeros(outSize, batchSize); + // Batch size better not change during an iteration... + if (outParameter.size() > 1) + { + Log::Fatal << "GRU<>::Forward(): batch size cannot change during a " + << "forward pass!" << std::endl; + } + + outParameter.clear(); + outParameter.emplace_back(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true); + + prevOutput = outParameter.begin(); + backIterator = outParameter.end(); + gradIterator = outParameter.end(); + } + + // Process the input linearly(zt, rt, ot). + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, input2GateModule)), + input2GateModule); + + // Process the output(zt, rt) linearly. + boost::apply_visitor(ForwardVisitor(*prevOutput, + boost::apply_visitor(outputParameterVisitor, output2GateModule)), + output2GateModule); + + // Merge the outputs(zt and rt). + output = (boost::apply_visitor(outputParameterVisitor, + input2GateModule).submat(0, 0, 2 * outSize - 1, batchSize - 1) + + boost::apply_visitor(outputParameterVisitor, output2GateModule)); + + // Pass the first outSize through inputGate(it). + boost::apply_visitor(ForwardVisitor(output.submat( + 0, 0, 1 * outSize - 1, batchSize - 1), boost::apply_visitor( + outputParameterVisitor, inputGateModule)), inputGateModule); + + // Pass the second through forgetGate. + boost::apply_visitor(ForwardVisitor(output.submat( + 1 * outSize, 0, 2 * outSize - 1, batchSize - 1), + boost::apply_visitor(outputParameterVisitor, forgetGateModule)), + forgetGateModule); + + arma::mat modInput = (boost::apply_visitor(outputParameterVisitor, + forgetGateModule) % *prevOutput); + + // Pass that through the outputHidden2GateModule. + boost::apply_visitor(ForwardVisitor(modInput, + boost::apply_visitor(outputParameterVisitor, outputHidden2GateModule)), + outputHidden2GateModule); + + // Merge for ot. + arma::mat outputH = boost::apply_visitor(outputParameterVisitor, + input2GateModule).submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1) + + boost::apply_visitor(outputParameterVisitor, outputHidden2GateModule); + + // Pass it through hiddenGate. + boost::apply_visitor(ForwardVisitor(outputH, + boost::apply_visitor(outputParameterVisitor, hiddenStateModule)), + hiddenStateModule); + + // Update the output (nextOutput): cmul1 + cmul2 + // Where cmul1 is input gate * prevOutput and + // cmul2 is (1 - input gate) * hidden gate. + output = (boost::apply_visitor(outputParameterVisitor, inputGateModule) + % (*prevOutput - boost::apply_visitor(outputParameterVisitor, + hiddenStateModule))) + boost::apply_visitor(outputParameterVisitor, + hiddenStateModule); + + forwardStep++; + if (forwardStep == rho) + { + forwardStep = 0; + if (!deterministic) + { + outParameter.emplace_back(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true); + prevOutput = --outParameter.end(); + } + else + { + *prevOutput = arma::mat(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true); + } + } + else if (!deterministic) + { + outParameter.push_back(output); + prevOutput = --outParameter.end(); + } + else + { + if (forwardStep == 1) + { + outParameter.clear(); + outParameter.push_back(output); + + prevOutput = outParameter.begin(); + } + else + { + *prevOutput = output; + } + } +} + +template +template +void GRU::Backward( + const arma::Mat& input, const arma::Mat& gy, arma::Mat& g) +{ + if (input.n_cols != batchSize) + { + batchSize = input.n_cols; + prevError.resize(3 * outSize, batchSize); + allZeros.zeros(outSize, batchSize); + // Batch size better not change during an iteration... + if (outParameter.size() > 1) + { + Log::Fatal << "GRU<>::Forward(): batch size cannot change during a " + << "forward pass!" << std::endl; + } + + outParameter.clear(); + outParameter.emplace_back(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true); + + prevOutput = outParameter.begin(); + backIterator = outParameter.end(); + gradIterator = outParameter.end(); + } + + arma::Mat gyLocal; + if ((outParameter.size() - backwardStep - 1) % rho != 0 && backwardStep != 0) + { + gyLocal = gy + boost::apply_visitor(deltaVisitor, output2GateModule); + } + else + { + gyLocal = arma::Mat(((arma::Mat&) gy).memptr(), gy.n_rows, + gy.n_cols, false, false); + } + + if (backIterator == outParameter.end()) + { + backIterator = --(--outParameter.end()); + } + + // Delta zt. + arma::mat dZt = gyLocal % (*backIterator - + boost::apply_visitor(outputParameterVisitor, + hiddenStateModule)); + + // Delta ot. + arma::mat dOt = gyLocal % (arma::ones(outSize, batchSize) - + boost::apply_visitor(outputParameterVisitor, inputGateModule)); + + // Delta of input gate. + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, inputGateModule), dZt, + boost::apply_visitor(deltaVisitor, inputGateModule)), + inputGateModule); + + // Delta of hidden gate. + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, hiddenStateModule), dOt, + boost::apply_visitor(deltaVisitor, hiddenStateModule)), + hiddenStateModule); + + // Delta of outputHidden2GateModule. + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, outputHidden2GateModule), + boost::apply_visitor(deltaVisitor, hiddenStateModule), + boost::apply_visitor(deltaVisitor, outputHidden2GateModule)), + outputHidden2GateModule); + + // Delta rt. + arma::mat dRt = boost::apply_visitor(deltaVisitor, outputHidden2GateModule) % + *backIterator; + + // Delta of forget gate. + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, forgetGateModule), dRt, + boost::apply_visitor(deltaVisitor, forgetGateModule)), + forgetGateModule); + + // Put delta zt. + prevError.submat(0, 0, 1 * outSize - 1, batchSize - 1) = boost::apply_visitor( + deltaVisitor, inputGateModule); + + // Put delta rt. + prevError.submat(1 * outSize, 0, 2 * outSize - 1, batchSize - 1) = + boost::apply_visitor(deltaVisitor, forgetGateModule); + + // Put delta ot. + prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1) = + boost::apply_visitor(deltaVisitor, hiddenStateModule); + + // Get delta ht - 1 for input gate and forget gate. + arma::mat prevErrorSubview = prevError.submat(0, 0, 2 * outSize - 1, + batchSize - 1); + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, input2GateModule), + prevErrorSubview, + boost::apply_visitor(deltaVisitor, output2GateModule)), + output2GateModule); + + // Add delta ht - 1 from hidden state. + boost::apply_visitor(deltaVisitor, output2GateModule) += + boost::apply_visitor(deltaVisitor, outputHidden2GateModule) % + boost::apply_visitor(outputParameterVisitor, forgetGateModule); + + // Add delta ht - 1 from ht. + boost::apply_visitor(deltaVisitor, output2GateModule) += gyLocal % + boost::apply_visitor(outputParameterVisitor, inputGateModule); + + // Get delta input. + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, input2GateModule), prevError, + boost::apply_visitor(deltaVisitor, input2GateModule)), + input2GateModule); + + backwardStep++; + backIterator--; + + g = boost::apply_visitor(deltaVisitor, input2GateModule); +} + +template +template +void GRU::Gradient( + const arma::Mat& input, + const arma::Mat& /* error */, + arma::Mat& /* gradient */) +{ + if (input.n_cols != batchSize) + { + batchSize = input.n_cols; + prevError.resize(3 * outSize, batchSize); + allZeros.zeros(outSize, batchSize); + // Batch size better not change during an iteration... + if (outParameter.size() > 1) + { + Log::Fatal << "GRU<>::Forward(): batch size cannot change during a " + << "forward pass!" << std::endl; + } + + outParameter.clear(); + outParameter.emplace_back(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true); + + prevOutput = outParameter.begin(); + backIterator = outParameter.end(); + gradIterator = outParameter.end(); + } + + if (gradIterator == outParameter.end()) + { + gradIterator = --(--outParameter.end()); + } + + boost::apply_visitor(GradientVisitor(input, prevError), input2GateModule); + + boost::apply_visitor(GradientVisitor( + *gradIterator, + prevError.submat(0, 0, 2 * outSize - 1, batchSize - 1)), + output2GateModule); + + boost::apply_visitor(GradientVisitor( + *gradIterator % boost::apply_visitor(outputParameterVisitor, + forgetGateModule), + prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1)), + outputHidden2GateModule); + + gradIterator--; +} + +template +void GRU::ResetCell(const size_t /* size */) +{ + outParameter.clear(); + outParameter.emplace_back(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true); + + prevOutput = outParameter.begin(); + backIterator = outParameter.end(); + gradIterator = outParameter.end(); + + forwardStep = 0; + backwardStep = 0; +} + +template +template +void GRU::serialize( + Archive& ar, const uint32_t /* version */) +{ + // If necessary, clean memory from the old model. + if (cereal::is_loading()) + { + boost::apply_visitor(deleteVisitor, input2GateModule); + boost::apply_visitor(deleteVisitor, output2GateModule); + boost::apply_visitor(deleteVisitor, outputHidden2GateModule); + boost::apply_visitor(deleteVisitor, inputGateModule); + boost::apply_visitor(deleteVisitor, forgetGateModule); + boost::apply_visitor(deleteVisitor, hiddenStateModule); + } + + ar(CEREAL_NVP(inSize)); + ar(CEREAL_NVP(outSize)); + ar(CEREAL_NVP(rho)); + + ar(CEREAL_VARIANT_POINTER(input2GateModule)); + ar(CEREAL_VARIANT_POINTER(output2GateModule)); + ar(CEREAL_VARIANT_POINTER(outputHidden2GateModule)); + ar(CEREAL_VARIANT_POINTER(inputGateModule)); + ar(CEREAL_VARIANT_POINTER(forgetGateModule)); + ar(CEREAL_VARIANT_POINTER(hiddenStateModule)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/hard_tanh.hpp b/src/mlpack/methods/ann/layer/hard_tanh.hpp similarity index 71% rename from src/mlpack/methods/ann/layer/not_adapted/hard_tanh.hpp rename to src/mlpack/methods/ann/layer/hard_tanh.hpp index b214ff095b..3c49add924 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/hard_tanh.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh.hpp @@ -14,8 +14,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -39,14 +37,16 @@ namespace ann /** Artificial Neural Network. */ { * \right. * @f} * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class HardTanHType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class HardTanH { public: /** @@ -57,10 +57,7 @@ class HardTanHType : public Layer * @param maxValue Range of the linear region maximum value. * @param minValue Range of the linear region minimum value. */ - HardTanHType(const double maxValue = 1, const double minValue = -1); - - //! Clone the HardTanHType object. This handles polymorphism correctly. - HardTanHType* Clone() const { return new HardTanHType(*this); } + HardTanH(const double maxValue = 1, const double minValue = -1); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -69,6 +66,7 @@ class HardTanHType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -80,7 +78,20 @@ class HardTanHType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const DataType& input, + const DataType& gy, + DataType& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the maximum value. double const& MaxValue() const { return maxValue; } @@ -99,17 +110,18 @@ class HardTanHType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Maximum value for the HardTanH function. double maxValue; //! Minimum value for the HardTanH function. double minValue; -}; // class HardTanHType - -// Convenience typedefs. - -// Standard HardTanH layer. -typedef HardTanHType HardTanH; +}; // class HardTanH } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/hard_tanh_impl.hpp b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp similarity index 71% rename from src/mlpack/methods/ann/layer/not_adapted/hard_tanh_impl.hpp rename to src/mlpack/methods/ann/layer/hard_tanh_impl.hpp index 8778412d77..30eb8c2a1f 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/hard_tanh_impl.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp @@ -18,8 +18,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -HardTanHType::HardTanHType( +template +HardTanH::HardTanH( const double maxValue, const double minValue) : maxValue(maxValue), @@ -28,10 +28,12 @@ HardTanHType::HardTanHType( // Nothing to do here. } +template template -void HardTanHType::Forward( +void HardTanH::Forward( const InputType& input, OutputType& output) { + output = input; for (size_t i = 0; i < input.n_elem; ++i) { output(i) = (output(i) > maxValue ? maxValue : @@ -39,9 +41,10 @@ void HardTanHType::Forward( } } -template -void HardTanHType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) +template +template +void HardTanH::Backward( + const DataType& input, const DataType& gy, DataType& g) { g = gy; for (size_t i = 0; i < input.n_elem; ++i) @@ -53,14 +56,12 @@ void HardTanHType::Backward( } } -template +template template -void HardTanHType::serialize( +void HardTanH::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(maxValue)); ar(CEREAL_NVP(minValue)); } diff --git a/src/mlpack/methods/ann/layer/not_adapted/hardshrink.hpp b/src/mlpack/methods/ann/layer/hardshrink.hpp similarity index 67% rename from src/mlpack/methods/ann/layer/not_adapted/hardshrink.hpp rename to src/mlpack/methods/ann/layer/hardshrink.hpp index df6c61756c..5817acb713 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/hardshrink.hpp +++ b/src/mlpack/methods/ann/layer/hardshrink.hpp @@ -17,8 +17,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artifical Neural Network. */ { @@ -39,28 +37,23 @@ namespace ann /** Artifical Neural Network. */ { * \f} * * \f$\lambda\f$ is set to 0.5 by default. - * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). */ -template -class HardShrinkType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class HardShrink { public: /** * Create HardShrink object using specified hyperparameter lambda. * - * @param lambda Is calculated by multiplying the noise level sigma of the - * input(noisy image) and a coefficient 'a' which is one of the training - * parameters. Default value of lambda is 0.5. + * @param lambda Is calculated by multiplying the + * noise level sigma of the input(noisy image) and a + * coefficient 'a' which is one of the training parameters. + * Default value of lambda is 0.5. */ - HardShrinkType(const double lambda = 0.5); - - //! Clone the HardShrinkType object. This handles polymorphism correctly. - HardShrinkType* Clone() const { return new HardShrinkType(*this); } + HardShrink(const double lambda = 0.5); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -69,6 +62,7 @@ class HardShrinkType : public Layer * @param input Input data used for evaluating the Hard Shrink function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -80,26 +74,42 @@ class HardShrinkType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const DataType& input, + DataType& gy, + DataType& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the hyperparameter lambda. double const& Lambda() const { return lambda; } //! Modify the hyperparameter lambda. double& Lambda() { return lambda; } - //! Serialize the layer. + /** + * Serialize the layer. + */ template void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Locally-stored hyperparameter lambda. double lambda; -}; // class HardShrinkType - -// Convenience typedefs. - -// Standard HardShrink layer. -typedef HardShrinkType HardShrink; +}; // class HardShrink } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/hardshrink_impl.hpp b/src/mlpack/methods/ann/layer/hardshrink_impl.hpp similarity index 63% rename from src/mlpack/methods/ann/layer/not_adapted/hardshrink_impl.hpp rename to src/mlpack/methods/ann/layer/hardshrink_impl.hpp index b1098cbb32..5969a281dd 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/hardshrink_impl.hpp +++ b/src/mlpack/methods/ann/layer/hardshrink_impl.hpp @@ -20,35 +20,37 @@ namespace ann /** Artificial Neural Network. */ { // This constructor is called for Hard Shrink activation function. // 'lambda' is a hyperparameter. -template -HardShrinkType::HardShrinkType(const double lambda) : +template +HardShrink::HardShrink(const double lambda) : lambda(lambda) { // Nothing to do here. } +template template -void HardShrinkType::Forward( +void HardShrink::Forward( const InputType& input, OutputType& output) { output = ((input > lambda) + (input < -lambda)) % input; } -template -void HardShrinkType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) +template +template +void HardShrink::Backward( + const DataType& input, DataType& gy, DataType& g) { - g = gy % (arma::ones(arma::size(input)) - (input == 0)); + DataType derivative; + derivative = (arma::ones(arma::size(input)) - (input == 0)); + g = gy % derivative; } -template +template template -void HardShrinkType::serialize( +void HardShrink::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(lambda)); } diff --git a/src/mlpack/methods/ann/layer/highway.hpp b/src/mlpack/methods/ann/layer/highway.hpp new file mode 100644 index 0000000000..526a434d23 --- /dev/null +++ b/src/mlpack/methods/ann/layer/highway.hpp @@ -0,0 +1,270 @@ +/** + * @file methods/ann/layer/highway.hpp + * @author Konstantin Sidorov + * @author Saksham Bansal + * + * Definition of the Highway layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP +#define MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP + +#include + +#include "../visitor/delete_visitor.hpp" +#include "../visitor/delta_visitor.hpp" +#include "../visitor/output_height_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" +#include "../visitor/output_width_visitor.hpp" + +#include "layer_types.hpp" +#include "add_merge.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Implementation of the Highway layer. The Highway class can vary its behavior + * between that of feed-forward fully connected network container and that + * of a layer which simply passes its inputs through depending on the transform + * gate. Note that the size of the input and output matrices of this class + * should be equal. + * + * For more information, refer the following paper. + * + * @code + * @article{Srivastava2015, + * author = {Rupesh Kumar Srivastava, Klaus Greff, Jurgen Schmidhuber}, + * title = {Training Very Deep Networks}, + * journal = {Advances in Neural Information Processing Systems}, + * year = {2015}, + * url = {https://arxiv.org/abs/1507.06228}, + * } + * @endcode + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename... CustomLayers> +class Highway +{ + public: + //! Create the Highway object. + Highway(); + + /** + * Create the Highway object. + * + * @param inSize The number of input units. + * @param model Expose all the network modules. + */ + Highway(const size_t inSize, const bool model = true); + + //! Destroy the Highway object. + ~Highway(); + + /** + * Reset the layer parameter. + */ + void Reset(); + + /** + * Ordinary feed-forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const arma::Mat& input, arma::Mat& output); + + /** + * Ordinary feed-backward pass of a neural network, calculating the function + * f(x) by propagating x backwards through f. Using the results from the + * feed-forward pass. + * + * @param * (input) The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + /** + * Calculate the gradient using the output delta and the input activation. + * + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. + */ + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); + + /** + * Add a new module to the model. + * + * @param args The layer parameter. + */ + template + void Add(Args... args) + { + network.push_back(new LayerType(args...)); + networkOwnerships.push_back(true); + } + + /** + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) + { + network.push_back(layer); + networkOwnerships.push_back(false); + } + + //! Return the modules of the model. + std::vector >& Model() + { + if (model) + { + return network; + } + + return empty; + } + + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the number of input units. + size_t InSize() const { return inSize; } + + //! Get the shape of the input. + size_t InputShape() const + { + return inSize; + } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const uint32_t /* version */); + + private: + //! Locally-stored number of input units. + size_t inSize; + + //! Parameter which indicates if the modules should be exposed. + bool model; + + //! Indicator if we already initialized the model. + bool reset; + + //! Locally-stored network modules. + std::vector > network; + + //! The list of network modules we are responsible for. + std::vector networkOwnerships; + + //! Locally-stored empty list of modules. + std::vector > empty; + + //! Locally-stored weight object. + OutputDataType weights; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Weights for transformation of output. + OutputDataType transformWeight; + + //! Bias for transformation of output. + OutputDataType transformBias; + + //! Locally-stored transform gate parameters. + OutputDataType transformGate; + + //! Locally-stored transform gate activation. + OutputDataType transformGateActivation; + + //! Locally-stored transform gate error. + OutputDataType transformGateError; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! The input width. + size_t width; + + //! The input height. + size_t height; + + //! The normal output without highway network. + OutputDataType networkOutput; + + //! Locally-stored delta visitor. + DeltaVisitor deltaVisitor; + + //! Locally-stored output parameter visitor. + OutputParameterVisitor outputParameterVisitor; + + //! Locally-stored delete visitor. + DeleteVisitor deleteVisitor; + + //! Locally-stored output width visitor. + OutputWidthVisitor outputWidthVisitor; + + //! Locally-stored output height visitor. + OutputHeightVisitor outputHeightVisitor; +}; // class Highway + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "highway_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/highway_impl.hpp b/src/mlpack/methods/ann/layer/highway_impl.hpp new file mode 100644 index 0000000000..00418e03a3 --- /dev/null +++ b/src/mlpack/methods/ann/layer/highway_impl.hpp @@ -0,0 +1,238 @@ +/** + * @file methods/ann/layer/highway_impl.hpp + * @author Konstantin Sidorov + * @author Saksham Bansal + * + * Implementation of Highway layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_HIGHWAY_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_HIGHWAY_IMPL_HPP + +// In case it hasn't yet been included. +#include "highway.hpp" + +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" +#include "../visitor/set_input_height_visitor.hpp" +#include "../visitor/set_input_width_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +Highway::Highway() : + inSize(0), + model(true), + reset(false), + width(0), + height(0) +{ + // Nothing to do here. +} + +template< + typename InputDataType, typename OutputDataType, typename... CustomLayers> +Highway::Highway( + const size_t inSize, + const bool model) : + inSize(inSize), + model(model), + reset(false), + width(0), + height(0) +{ + weights.set_size(inSize * inSize + inSize, 1); +} + +template +Highway::~Highway() +{ + if (!model) + { + for (size_t i = 0; i < network.size(); ++i) + { + if (networkOwnerships[i]) + boost::apply_visitor(deleteVisitor, network[i]); + } + } +} + +template +void Highway::Reset() +{ + transformWeight = arma::mat(weights.memptr(), inSize, inSize, false, false); + transformBias = arma::mat(weights.memptr() + transformWeight.n_elem, + inSize, 1, false, false); +} + +template +template +void Highway::Forward( + const arma::Mat& input, arma::Mat& output) +{ + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network.front())), + network.front()); + + if (!reset) + { + if (boost::apply_visitor(outputWidthVisitor, network.front()) != 0) + { + width = boost::apply_visitor(outputWidthVisitor, network.front()); + } + + if (boost::apply_visitor(outputHeightVisitor, network.front()) != 0) + { + height = boost::apply_visitor(outputHeightVisitor, network.front()); + } + } + + for (size_t i = 1; i < network.size(); ++i) + { + if (!reset) + { + // Set the input width. + boost::apply_visitor(SetInputWidthVisitor(width), network[i]); + + // Set the input height. + boost::apply_visitor(SetInputHeightVisitor(height), network[i]); + } + + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[i - 1]), + boost::apply_visitor(outputParameterVisitor, network[i])), + network[i]); + + if (!reset) + { + // Get the output width. + if (boost::apply_visitor(outputWidthVisitor, network[i]) != 0) + { + width = boost::apply_visitor(outputWidthVisitor, network[i]); + } + + // Get the output height. + if (boost::apply_visitor(outputHeightVisitor, network[i]) != 0) + { + height = boost::apply_visitor(outputHeightVisitor, network[i]); + } + } + } + if (!reset) + { + reset = true; + } + + output = boost::apply_visitor(outputParameterVisitor, network.back()); + + if (arma::size(output) != arma::size(input)) + { + Log::Fatal << "The sizes of the output and input matrices of the Highway" + << " network should be equal. Please examine the network layers."; + } + + transformGate = transformWeight * input; + transformGate.each_col() += transformBias; + transformGateActivation = 1.0 /(1 + arma::exp(-transformGate)); + inputParameter = input; + networkOutput = output; + output = (output % transformGateActivation) + + (input % (1 - transformGateActivation)); +} + +template +template +void Highway::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) +{ + arma::Mat gyTransform = gy % transformGateActivation; + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network.back()), + gyTransform, + boost::apply_visitor(deltaVisitor, network.back())), + network.back()); + + for (size_t i = 2; i < network.size() + 1; ++i) + { + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]), + boost::apply_visitor(deltaVisitor, + network[network.size() - i])), network[network.size() - i]); + } + + g = boost::apply_visitor(deltaVisitor, network.front()); + + transformGateError = gy % (networkOutput - inputParameter) % + transformGateActivation % (1.0 - transformGateActivation); + g += transformWeight.t() * transformGateError; + g += gy % (1 - transformGateActivation); +} + +template +template +void Highway::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) +{ + arma::Mat errorTransform = error % transformGateActivation; + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - 2]), + errorTransform), network.back()); + + for (size_t i = 2; i < network.size(); ++i) + { + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i - 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1])), + network[network.size() - i]); + } + + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, network[1])), network.front()); + + gradient.submat(0, 0, transformWeight.n_elem - 1, 0) = arma::vectorise( + transformGateError * input.t()); + gradient.submat(transformWeight.n_elem, 0, gradient.n_elem - 1, 0) = + arma::sum(transformGateError, 1); +} + +template +template +void Highway::serialize( + Archive& ar, const uint32_t /* version */) +{ + // If loading, delete the old layers and set size for weights. + if (cereal::is_loading()) + { + for (LayerTypes& layer : network) + { + boost::apply_visitor(deleteVisitor, layer); + } + weights.set_size(inSize * inSize + inSize, 1); + } + + ar(CEREAL_NVP(model)); + ar(CEREAL_VECTOR_VARIANT_POINTER(network)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/instance_norm.hpp b/src/mlpack/methods/ann/layer/instance_norm.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/instance_norm.hpp rename to src/mlpack/methods/ann/layer/instance_norm.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/instance_norm_impl.hpp b/src/mlpack/methods/ann/layer/instance_norm_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/instance_norm_impl.hpp rename to src/mlpack/methods/ann/layer/instance_norm_impl.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/isrlu.hpp b/src/mlpack/methods/ann/layer/isrlu.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/isrlu.hpp rename to src/mlpack/methods/ann/layer/isrlu.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/isrlu_impl.hpp b/src/mlpack/methods/ann/layer/isrlu_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/isrlu_impl.hpp rename to src/mlpack/methods/ann/layer/isrlu_impl.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/join.hpp b/src/mlpack/methods/ann/layer/join.hpp similarity index 61% rename from src/mlpack/methods/ann/layer/not_adapted/join.hpp rename to src/mlpack/methods/ann/layer/join.hpp index ee5a7acd00..38286a9508 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/join.hpp +++ b/src/mlpack/methods/ann/layer/join.hpp @@ -17,30 +17,24 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -// TODO: should we clarify the comments? This seems to join together points of -// a different batch -// TODO: I don't understand this layer well enough to update it... /** * Implementation of the Join module class. The Join class accumulates * the output of various modules. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template< - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class JoinType : public Layer +class Join { public: - //! Create the JoinType object. - JoinType(); - - //! Clone the JoinType object. This handles polymorphism correctly. - JoinType* Clone() const { return new JoinType(*this); } + //! Create the Join object. + Join(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -49,6 +43,7 @@ class JoinType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -60,19 +55,20 @@ class JoinType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); - // This layer simply flattens its input into a vector. - const std::vector OutputDimensions() const - { - // TODO: it's not clear what to do here - std::vector result(inputDimensions.size(), 0); - result[0] = std::accumulate(inputDimensions.begin(), inputDimensions.end(), - 0); - return result; - } + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } /** * Serialize the layer. @@ -86,10 +82,13 @@ class JoinType : public Layer //! Locally-stored number of input cols. size_t inSizeCols; -}; // class JoinType -//Standard Join layer. -typedef JoinType Join; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class Join } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/join_impl.hpp b/src/mlpack/methods/ann/layer/join_impl.hpp similarity index 63% rename from src/mlpack/methods/ann/layer/not_adapted/join_impl.hpp rename to src/mlpack/methods/ann/layer/join_impl.hpp index aef5a9cc5a..a886d7ac44 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/join_impl.hpp +++ b/src/mlpack/methods/ann/layer/join_impl.hpp @@ -18,16 +18,17 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -JoinType::JoinType() : +template +Join::Join() : inSizeRows(0), inSizeCols(0) { // Nothing to do here. } +template template -void JoinType::Forward( +void Join::Forward( const InputType& input, OutputType& output) { inSizeRows = input.n_rows; @@ -35,24 +36,23 @@ void JoinType::Forward( output = arma::vectorise(input); } -template -void JoinType::Backward( - const InputType& /* input */, - const OutputType& gy, - OutputType& g) +template +template +void Join::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - g = OutputType(((OutputType&) gy).memptr(), inSizeRows, inSizeCols, false, + g = arma::mat(((arma::Mat&) gy).memptr(), inSizeRows, inSizeCols, false, false); } -template +template template -void JoinType::serialize( +void Join::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(inSizeRows)); ar(CEREAL_NVP(inSizeCols)); } diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index c3bfc6a593..8aca0ad270 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -2,7 +2,7 @@ * @file methods/ann/layer/layer.hpp * @author Marcus Edel * - * Base class for neural network layers. + * This includes various layers to construct a model. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -12,312 +12,73 @@ #ifndef MLPACK_METHODS_ANN_LAYER_LAYER_HPP #define MLPACK_METHODS_ANN_LAYER_LAYER_HPP -namespace mlpack { -namespace ann { - -/** - * A layer is an abstract class implementing common neural networks operations, - * such as convolution, batch norm, etc. These operations require managing - * weights, losses, updates, and inter-layer connectivity. - * - * Users will just instantiate a layer by inherited from the abstract class and - * implement the layer specific methods. It is recommend that descendants of - * Layer implement the following methods: - * - * - Constructor: Defines custom layer attributes, and creates layer state - * variables. - * - * - Forward(input, output): Performs the forward logic of applying the layer - * to the input object and storing the result in the output object. - * - * - Backward(input, gy, g): Performs a backpropagation step through the layer, - * with respect to the given input. - * - * - Gradient(input, error, gradient): Computing the gradient of the layer with - * respect to its own input. - * - * The memory for the layer's parameters (weights and biases) is not allocated - * by the layer itself, instead it is allocated by the network that the layer - * belongs to, and passed to the layer when it needs to use it. - * - * See the linear layer implementation for a basic example. It's a layer with - * two variables, w and b, that returns y = w * x + b. It shows how to implement - * Forward(), Backward() and Gradient(). The weights of the layers are tracked - * in layer.Parameters(). - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. - */ -template -class Layer -{ - public: - //! Default constructor. - Layer() : validOutputDimensions(false), training(false) - { /* Nothing to do here */ } - - //! Default deconstructor. - virtual ~Layer() { /* Nothing to do here */ } - - //! Copy constructor. This is not responsible for copying weights! - Layer(const Layer& layer) : - inputDimensions(layer.inputDimensions), - outputDimensions(layer.outputDimensions), - validOutputDimensions(layer.validOutputDimensions), - training(layer.training) - { /* Nothing to do here */ } - - //! Make a copy of the object. - virtual Layer* Clone() const = 0; - - //! Move constructor. This is not responsible for moving weights! - Layer(Layer&& layer) : - inputDimensions(std::move(layer.inputDimensions)), - outputDimensions(std::move(layer.outputDimensions)), - validOutputDimensions(std::move(layer.validOutputDimensions)), - training(std::move(layer.training)) - { /* Nothing to do here */ } - - //! Copy assignment operator. This is not responsible for copying weights! - virtual Layer& operator=(const Layer& layer) - { - if (&layer != this) - { - inputDimensions = layer.inputDimensions; - outputDimensions = layer.outputDimensions; - validOutputDimensions = layer.validOutputDimensions; - training = layer.training; - } - - return *this; - } - - //! Move assignment operator. This is not responsible for moving weights! - virtual Layer& operator=(Layer&& layer) - { - if (&layer != this) - { - inputDimensions = std::move(layer.inputDimensions); - outputDimensions = std::move(layer.outputDimensions); - validOutputDimensions = std::move(layer.validOutputDimensions); - training = std::move(layer.training); - } - - return *this; - } - - /** - * Takes an input object, and computes the corresponding output of the layer. - * In general input and output are matrices. However, some special layers like - * table layers might expect something else. Please, refer to each layer - * specification for further information. - * - * @param * (input) Input data used for evaluating the specified layer. - * @param * (output) Resulting output. - */ - virtual void Forward(const MatType& /* input */, - MatType& /* output */) - { /* Nothing to do here */ } - - /** - * Takes an input and output object, and computes the corresponding loss of - * the layer. In general input and output are matrices. However, some special - * layers like table layers might expect something else. Please, refer to each - * layer specification for further information. - * - * @param * (input) Input data used for evaluating the specified layer. - * @param * (output) Resulting output. - */ - virtual void Forward(const MatType& /* input */, - const MatType& /* output */) - { /* Nothing to do here */ } - - /** - * Performs a backpropagation step through the layer, with respect to the - * given input. In general this method makes the assumption Forward(input, - * output) has been called before, with the same input. If you do not respect - * this rule, Backward(input, gy, g) might compute incorrect results. - * - * In general input and gy and g are matrices. However, some special - * sub-classes like table layers might expect something else. Please, refer to - * each module specification for further information. - * - * A backpropagation step consist of computing of computing the gradient - * output input with respect to the output of the layer and given error. - * - * During the backward pass our goal is to use 'gy' in order to compute the - * downstream gradients (g). We assume that the upstream gradient (gy) has - * already been computed and is passed to the layer. - * - * @param * (input) The propagated input activation. - * @param * (gy) The backpropagated error. - * @param * (g) The calculated gradient. - */ - virtual void Backward(const MatType& /* input */, - const MatType& /* gy */, - MatType& /* g */) - { /* Nothing to do here */ } - - /** - * Computing the gradient of the layer with respect to its own input. This is - * returned in gradient. - * - * The layer parameters (weights and biases) are updated accordingly using the - * computed gradient not by the layer itself, instead they are updated by the - * network that holds the instantiated layer. - * - * @param * (input) The input parameter used for calculating the gradient. - * @param * (error) The calculated error. - * @param * (gradient) The calculated gradient. - */ - virtual void Gradient(const MatType& /* input */, - const MatType& /* error */, - MatType& /* gradient */) - { /* Nothing to do here */ } - - /** - * Reset the layer parameter. The method is called to assigned the allocated - * memory to the internal layer parameters like weights and biases. The method - * should be called before the first call of Forward(input, output). If you - * do not respect this rule, Forward(input, output) and Backward(input, gy, g) - * might compute incorrect results. - * - * @param weightsPtr This pointer should be used as the first element of the - * memory that is allocated for this layer. In general, SetWeights() - * implementations should use MakeAlias() with weightsPtr to wrap the - * weights of a layer. - */ - virtual void SetWeights(typename MatType::elem_type* /* weightsPtr */) { } - - /** - * Get the total number of trainable weights in the layer. - */ - virtual size_t WeightSize() const { return 0; } - - /** - * Get whether the layer is currently in training mode. - * - * @note During network training, this should be set to `true` for each layer - * in the network, and when predicting/testing the network, this should be set - * to `false`. (This is handled automatically by the `FFN` class and other - * related classes.) - */ - virtual bool const& Training() const { return training; } - - /** - * Modify whether the layer is currently in training mode. - * - * @note During network training, this should be set to `true` for each layer - * in the network, and when predicting/testing the network, this should be set - * to `false`. (This is handled automatically by the `FFN` class and other - * related classes.) - */ - virtual bool& Training() { return training; } - - //! Get the layer loss. Overload this if the layer should add any extra loss - //! to the loss function when computing the objective. (TODO: better comment) - virtual double Loss() { return 0; } - - //! Get the input dimensions. - const std::vector& InputDimensions() const { return inputDimensions; } - //! Modify the input dimensions. - std::vector& InputDimensions() - { - validOutputDimensions = false; - return inputDimensions; - } - - //! Get the output dimensions. - const std::vector& OutputDimensions() - { - if (!validOutputDimensions) - { - this->ComputeOutputDimensions(); - validOutputDimensions = true; - } - - return outputDimensions; - } - - //! Get the parameters. - virtual const MatType& Parameters() const - { - throw std::invalid_argument("Layer::Parameters(): cannot access parameters " - "of a layer with no weights!"); - } - //! Set the parameters. - virtual MatType& Parameters() - { - throw std::invalid_argument("Layer::Parameters(): cannot modify parameters " - "of a layer with no weights!"); - } - - //! Compute the output dimensions. This should be overloaded if the layer is - //! meant to work on higher-dimensional objects. When this is called, it is a - //! safe assumption that InputDimensions() is correct. - virtual void ComputeOutputDimensions() - { - // The default implementation is to assume that the output size is the same - // as the input. - outputDimensions = inputDimensions; - } - - //! Get the number of elements in the output from this layer. This cannot be - //! overloaded! Overload `ComputeOutputDimensions()` instead. - virtual size_t OutputSize() final - { - if (!validOutputDimensions) - { - this->ComputeOutputDimensions(); - validOutputDimensions = true; - } - - size_t outputSize = 1; - for (size_t i = 0; i < this->outputDimensions.size(); ++i) - outputSize *= this->outputDimensions[i]; - return outputSize; - } - - //! Serialize the layer. - template - void serialize(Archive& ar, const uint32_t /* version */) - { - ar(CEREAL_NVP(inputDimensions)); - ar(CEREAL_NVP(outputDimensions)); - ar(CEREAL_NVP(validOutputDimensions)); - ar(CEREAL_NVP(training)); - - // Note that layer weights are serialized by the FFN! - } - - protected: - /** - * Logical input dimensions of each point. Although each point given to ! - * `Forward()` will be represented as a column in a matrix, logically - * speaking it can be a higher-order tensor. So, for instance, if the point - * is 2-dimensional images of size 10x10, `Forward()` will contain columns - * with 100 rows, and `inputDimensions` will be `{10, 10}`. This generalizes - * to higher dimensions. - */ - std::vector inputDimensions; - - /** - * Logical output dimensions of each point. If the layer only performs - * elementwise operations, this is most likely equal to `inputDimensions`; but - * if the layer performs more complicated transformations, it may be - * different. - */ - std::vector outputDimensions; - - //! This is `true` if `ComputeOutputDimensions()` has been called, and - //! `outputDimensions` can be considered to be up-to-date. - bool validOutputDimensions; - - //! If true, the layer is in training mode; otherwise, it is in testing mode. - bool training; -}; - -} // namespace ann -} // namespace mlpack +#include "add.hpp" +#include "adaptive_max_pooling.hpp" +#include "adaptive_mean_pooling.hpp" +#include "add_merge.hpp" +#include "alpha_dropout.hpp" +#include "atrous_convolution.hpp" +#include "base_layer.hpp" +#include "batch_norm.hpp" +#include "bicubic_interpolation.hpp" +#include "bilinear_interpolation.hpp" +#include "c_relu.hpp" +#include "celu.hpp" +#include "concat_performance.hpp" +#include "concat.hpp" +#include "concatenate.hpp" +#include "constant.hpp" +#include "convolution.hpp" +#include "dropconnect.hpp" +#include "dropout.hpp" +#include "elu.hpp" +#include "fast_lstm.hpp" +#include "flatten_t_swish.hpp" +#include "flexible_relu.hpp" +#include "glimpse.hpp" +#include "gru.hpp" +#include "hard_tanh.hpp" +#include "hardshrink.hpp" +#include "highway.hpp" +#include "instance_norm.hpp" +#include "join.hpp" +#include "layer_norm.hpp" +#include "layer_types.hpp" +#include "leaky_relu.hpp" +#include "linear.hpp" +#include "linear_no_bias.hpp" +#include "linear3d.hpp" +#include "log_softmax.hpp" +#include "lookup.hpp" +#include "lp_pooling.hpp" +#include "lstm.hpp" +#include "max_pooling.hpp" +#include "mean_pooling.hpp" +#include "minibatch_discrimination.hpp" +#include "multihead_attention.hpp" +#include "multiply_constant.hpp" +#include "multiply_merge.hpp" +#include "nearest_interpolation.hpp" +#include "noisylinear.hpp" +#include "padding.hpp" +#include "parametric_relu.hpp" +#include "pixel_shuffle.hpp" +#include "positional_encoding.hpp" +#include "recurrent_attention.hpp" +#include "recurrent.hpp" +#include "reinforce_normal.hpp" +#include "relu6.hpp" +#include "reparametrization.hpp" +#include "select.hpp" +#include "sequential.hpp" +#include "softshrink.hpp" +#include "softmax.hpp" +#include "softmin.hpp" +#include "spatial_dropout.hpp" +#include "subview.hpp" +#include "transposed_convolution.hpp" +#include "virtual_batch_norm.hpp" +#include "vr_class_reward.hpp" +#include "weight_norm.hpp" #endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/layer_norm.hpp b/src/mlpack/methods/ann/layer/layer_norm.hpp similarity index 67% rename from src/mlpack/methods/ann/layer/not_adapted/layer_norm.hpp rename to src/mlpack/methods/ann/layer/layer_norm.hpp index 822e09fc7d..c22408c221 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/layer_norm.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm.hpp @@ -53,20 +53,20 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class LayerNormType : public Layer +class LayerNorm { public: - //! Create the LayerNormType object. - LayerNormType(); + //! Create the LayerNorm object. + LayerNorm(); /** * Create the LayerNorm object for a specified number of input units. @@ -74,10 +74,7 @@ class LayerNormType : public Layer * @param size The number of input units. * @param eps The epsilon added to variance to ensure numerical stability. */ - LayerNormType(const size_t size, const double eps = 1e-8); - - //! Clone the LayerNormType object. This handles polymorphism correctly. - LayerNormType* Clone() const { return new LayerNormType(*this); } + LayerNorm(const size_t size, const double eps = 1e-8); /** * Reset the layer parameters. @@ -92,7 +89,8 @@ class LayerNormType : public Layer * @param input Input data for the layer. * @param output Resulting output activations. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Backward pass through the layer. @@ -101,9 +99,10 @@ class LayerNormType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activations. @@ -112,20 +111,36 @@ class LayerNormType : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the mean across single training data. - OutputType Mean() { return mean; } + OutputDataType Mean() { return mean; } //! Get the variance across single training data. - OutputType Variance() { return variance; } + OutputDataType Variance() { return variance; } //! Get the number of input units. size_t InSize() const { return size; } @@ -133,7 +148,11 @@ class LayerNormType : public Layer //! Get the value of epsilon. double Epsilon() const { return eps; } - const size_t WeightSize() const { return 2 * size; } + //! Get the shape of the input. + size_t InputShape() const + { + return size; + } /** * Serialize the layer. @@ -152,29 +171,35 @@ class LayerNormType : public Layer bool loading; //! Locally-stored scale parameter. - OutputType gamma; + OutputDataType gamma; //! Locally-stored shift parameter. - OutputType beta; + OutputDataType beta; //! Locally-stored parameters. - OutputType weights; + OutputDataType weights; //! Locally-stored mean object. - OutputType mean; + OutputDataType mean; //! Locally-stored variance object. - OutputType variance; + OutputDataType variance; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; //! Locally-stored normalized input. - OutputType normalized; + OutputDataType normalized; //! Locally-stored zero mean input. - OutputType inputMean; -}; // class LayerNormType - -// Standard LayerNorm type -typedef LayerNormType LayerNorm; + OutputDataType inputMean; +}; // class LayerNorm } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/layer_norm_impl.hpp b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp similarity index 60% rename from src/mlpack/methods/ann/layer/not_adapted/layer_norm_impl.hpp rename to src/mlpack/methods/ann/layer/layer_norm_impl.hpp index 3381bc93fe..ae4ced5cf8 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/layer_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm_impl.hpp @@ -20,8 +20,8 @@ namespace mlpack { namespace ann { /** Artificial Neural Network. */ -template -LayerNormType::LayerNormType() : +template +LayerNorm::LayerNorm() : size(0), eps(1e-8), loading(false) @@ -29,8 +29,8 @@ LayerNormType::LayerNormType() : // Nothing to do here. } -template -LayerNormType::LayerNormType( +template +LayerNorm::LayerNorm( const size_t size, const double eps) : size(size), eps(eps), @@ -39,12 +39,11 @@ LayerNormType::LayerNormType( weights.set_size(size + size, 1); } -template -void LayerNormType::SetWeights( - typename OutputType::elem_type* weightsPtr) +template +void LayerNorm::Reset() { - gamma = OutputType(weightsPtr, size, 1, false, false); - beta = OutputType(weightsPtr + gamma.n_elem, size, 1, false, false); + gamma = arma::mat(weights.memptr(), size, 1, false, false); + beta = arma::mat(weights.memptr() + gamma.n_elem, size, 1, false, false); if (!loading) { @@ -55,9 +54,10 @@ void LayerNormType::SetWeights( loading = false; } -template -void LayerNormType::Forward( - const InputType& input, OutputType& output) +template +template +void LayerNorm::Forward( + const arma::Mat& input, arma::Mat& output) { mean = arma::mean(input, 0); variance = arma::var(input, 1, 0); @@ -75,17 +75,18 @@ void LayerNormType::Forward( output.each_col() += beta; } -template -void LayerNormType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) +template +template +void LayerNorm::Backward( + const arma::Mat& input, const arma::Mat& gy, arma::Mat& g) { - const OutputType stdInv = 1.0 / arma::sqrt(variance + eps); + const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); // dl / dxhat. - const OutputType norm = gy.each_col() % gamma; + const arma::mat norm = gy.each_col() % gamma; // sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. - const OutputType var = arma::sum(norm % inputMean, 0) % + const arma::mat var = arma::sum(norm % inputMean, 0) % arma::pow(stdInv, 3.0) * -0.5; // dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + @@ -98,11 +99,12 @@ void LayerNormType::Backward( g.each_row() += arma::sum(norm.each_row() % -stdInv, 0) / input.n_rows; } -template -void LayerNormType::Gradient( - const InputType& /* input */, - const OutputType& error, - OutputType& gradient) +template +template +void LayerNorm::Gradient( + const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient) { gradient.set_size(size + size, 1); @@ -114,22 +116,22 @@ void LayerNormType::Gradient( arma::sum(error, 1); } -template +template template -void LayerNormType::serialize( +void LayerNorm::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(size)); - ar(CEREAL_NVP(eps)); - // Ensure that we don't set the values of the weights if we have already - // learned them. if (cereal::is_loading()) { + weights.set_size(size + size, 1); loading = true; } + + ar(CEREAL_NVP(eps)); + ar(CEREAL_NVP(gamma)); + ar(CEREAL_NVP(beta)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/layer_traits.hpp b/src/mlpack/methods/ann/layer/layer_traits.hpp new file mode 100644 index 0000000000..a6a447f43d --- /dev/null +++ b/src/mlpack/methods/ann/layer/layer_traits.hpp @@ -0,0 +1,130 @@ +/** + * @file methods/ann/layer/layer_traits.hpp + * @author Marcus Edel + * + * This provides the LayerTraits class, a template class to get information + * about various layers. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_LAYER_TRAITS_HPP +#define MLPACK_METHODS_ANN_LAYER_LAYER_TRAITS_HPP + +#include + +namespace mlpack { +namespace ann { + +/** + * This is a template class that can provide information about various layers. + * By default, this class will provide the weakest possible assumptions on + * layer, and each layer should override values as necessary. If a layer + * doesn't need to override a value, then there's no need to write a LayerTraits + * specialization for that class. + */ +template +class LayerTraits +{ + public: + /** + * This is true if the layer is a binary layer. + */ + static const bool IsBinary = false; + + /** + * This is true if the layer is an output layer. + */ + static const bool IsOutputLayer = false; + + /** + * This is true if the layer is a bias layer. + */ + static const bool IsBiasLayer = false; + + /* + * This is true if the layer is a LSTM layer. + **/ + static const bool IsLSTMLayer = false; + + /* + * This is true if the layer is a connection layer. + **/ + static const bool IsConnection = false; +}; + +// This gives us a HasGradientCheck type (where U is a function pointer) +// we can use with SFINAE to catch when a type has a Gradient(...) function. +HAS_MEM_FUNC(Gradient, HasGradientCheck); + +// This gives us a HasDeterministicCheck type (where U is a function +// pointer) we can use with SFINAE to catch when a type has a Deterministic() +// function. +HAS_MEM_FUNC(Deterministic, HasDeterministicCheck); + +// This gives us a HasParametersCheck type (where U is a function pointer) +// we can use with SFINAE to catch when a type has a Parameters() function. +HAS_MEM_FUNC(Parameters, HasParametersCheck); + +// This gives us a HasAddCheck type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a Add() function. +HAS_MEM_FUNC(Add, HasAddCheck); + +// This gives us a HasModelCheck type we can use with SFINAE to catch when +// a type has a function named Model. +HAS_ANY_METHOD_FORM(Model, HasModelCheck); + +// This gives us a HasLocationCheck type (where U is a function pointer) +// we can use with SFINAE to catch when a type has a Location() function. +HAS_MEM_FUNC(Location, HasLocationCheck); + +// This gives us a HasResetCheck type (where U is a function pointer) +// we can use with SFINAE to catch when a type has a Reset() function. +HAS_MEM_FUNC(Reset, HasResetCheck); + +// This gives us a HasResetCheck type (where U is a function pointer) +// we can use with SFINAE to catch when a type has a ResetCell() function. +HAS_MEM_FUNC(ResetCell, HasResetCellCheck); + +// This gives us a HasRewardCheck type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a Reward() function. +HAS_MEM_FUNC(Reward, HasRewardCheck); + +// This gives us a HasInputWidth type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a InputWidth() function. +HAS_MEM_FUNC(InputWidth, HasInputWidth); + +// This gives us a HasInputHeight type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a InputHeight() function. +HAS_MEM_FUNC(InputHeight, HasInputHeight); + +// This gives us a HasRho type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a Rho() function. +HAS_MEM_FUNC(Rho, HasRho); + +// This gives us a HasLoss type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a Loss() function. +HAS_MEM_FUNC(Loss, HasLoss); + +// This gives us a HasRunCheck type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a Run() function. +HAS_MEM_FUNC(Run, HasRunCheck); + +// This gives us a HasBiasCheck type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a Bias() function. +HAS_MEM_FUNC(Bias, HasBiasCheck); + +// This gives us a HasMaxIterationsC type (where U is a function pointer) +// we can use with SFINAE to catch when a type has a MaxIterations() function. +HAS_MEM_FUNC(MaxIterations, HasMaxIterations); + +// This gives us a HasInShapeCheck type we can use with SFINAE to catch when +// a type has a function named InputShape. +HAS_ANY_METHOD_FORM(InputShape, HasInputShapeCheck); + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index fb179b6457..a1970bd6e1 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -12,33 +12,62 @@ #ifndef MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP #define MLPACK_METHODS_ANN_LAYER_LAYER_TYPES_HPP -#include -#include -#include +#include -// Include each layer. +// Layer modules. #include #include #include +#include +#include +#include +#include +#include #include -#include -#include #include +#include +#include +#include +#include +#include +#include #include -#include +#include +#include #include #include #include -#include +#include +#include +#include #include +#include +#include +#include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include -// Convolution modes. +// Convolution modules. #include -#include #include +#include // Regularizers. #include @@ -46,7 +75,250 @@ // Loss function modules. #include -// Include definitions for polymorphic serialization. -#include +namespace mlpack { +namespace ann { + +template class BatchNorm; +template class DropConnect; +template class Glimpse; +template class LayerNorm; +template class LSTM; +template class GRU; +template class FastLSTM; +template class VRClassReward; +template class Concatenate; +template class Padding; +template class ReLU6; + +template +class Linear; + +template +class RBF; + +template +class LinearNoBias; + +template +class NoisyLinear; + +template +class Linear3D; + +template +class VirtualBatchNorm; + +template +class MiniBatchDiscrimination; + +template +class MultiheadAttention; + +template +class Reparametrization; + +template +class AddMerge; + +template +class Sequential; + +template +class Highway; + +template +class Recurrent; + +template +class Concat; + +template< + typename OutputLayerType, + typename InputDataType, + typename OutputDataType +> +class ConcatPerformance; + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +class Convolution; + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +class TransposedConvolution; + +template< + typename ForwardConvolutionRule, + typename BackwardConvolutionRule, + typename GradientConvolutionRule, + typename InputDataType, + typename OutputDataType +> +class AtrousConvolution; + +template< + typename InputDataType, + typename OutputDataType +> +class RecurrentAttention; + +template +class MultiplyMerge; + +template +class WeightNorm; + +template +class AdaptiveMaxPooling; + +template +class AdaptiveMeanPooling; + +using MoreTypes = boost::variant< + FlexibleReLU*, + Linear3D*, + LpPooling*, + PixelShuffle*, + ChannelShuffle*, + Glimpse*, + Highway*, + MultiheadAttention*, + Recurrent*, + RecurrentAttention*, + ReinforceNormal*, + ReLU6*, + Reparametrization*, + Select*, + SpatialDropout*, + Subview*, + VRClassReward*, + VirtualBatchNorm*, + RBF*, + BaseLayer*, + PositionalEncoding*, + ISRLU*, + BicubicInterpolation*, + NearestInterpolation*, + GroupNorm*, + InstanceNorm* +>; + +template +using LayerTypes = boost::variant< + AdaptiveMaxPooling*, + AdaptiveMeanPooling*, + Add*, + AddMerge*, + AlphaDropout*, + AtrousConvolution, + NaiveConvolution, + NaiveConvolution, + arma::mat, arma::mat>*, + BaseLayer*, + BaseLayer*, + BaseLayer*, + BaseLayer*, + BaseLayer*, + BatchNorm*, + BilinearInterpolation*, + CELU*, + Concat*, + Concatenate*, + ConcatPerformance, + arma::mat, arma::mat>*, + Constant*, + Convolution, + NaiveConvolution, + NaiveConvolution, arma::mat, arma::mat>*, + CReLU*, + DropConnect*, + Dropout*, + ELU*, + FastLSTM*, + GRU*, + HardTanH*, + Join*, + LayerNorm*, + LeakyReLU*, + Linear*, + LinearNoBias*, + LogSoftMax*, + Lookup*, + LSTM*, + MaxPooling*, + MeanPooling*, + MiniBatchDiscrimination*, + MultiplyConstant*, + MultiplyMerge*, + NegativeLogLikelihood*, + NoisyLinear*, + Padding*, + PReLU*, + Sequential*, + Sequential*, + Softmax*, + TransposedConvolution, + NaiveConvolution, + NaiveConvolution, arma::mat, arma::mat>*, + WeightNorm*, + MoreTypes, + CustomLayers*... +>; + +} // namespace ann +} // namespace mlpack #endif diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp index b275371de6..52b2896fca 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp @@ -16,8 +16,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -34,11 +32,16 @@ namespace ann /** Artificial Neural Network. */ { * \right. * @f} * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class LeakyReLUType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class LeakyReLU { public: /** @@ -46,24 +49,9 @@ class LeakyReLUType : public Layer * The non zero gradient can be adjusted by specifying the parameter * alpha in the range 0 to 1. Default (alpha = 0.03) * - * @param alpha Non zero gradient. + * @param alpha Non zero gradient */ - LeakyReLUType(const double alpha = 0.03); - - //! Clone the LeakyReLUType object. This handles polymorphism correctly. - LeakyReLUType* Clone() const { return new LeakyReLUType(*this); } - - // Virtual destructor. - virtual ~LeakyReLUType() { } - - //! Copy the given LeakyReLUType. - LeakyReLUType(const LeakyReLUType& other); - //! Take ownership of the given LeakyReLUType. - LeakyReLUType(LeakyReLUType&& other); - //! Copy the given LeakyReLUType. - LeakyReLUType& operator=(const LeakyReLUType& other); - //! Take ownership of the given LeakyReLUType. - LeakyReLUType& operator=(LeakyReLUType&& other); + LeakyReLU(const double alpha = 0.03); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -72,7 +60,8 @@ class LeakyReLUType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -83,27 +72,43 @@ class LeakyReLUType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& input, const MatType& gy, MatType& g); + template + void Backward(const DataType& input, const DataType& gy, DataType& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the non zero gradient. double const& Alpha() const { return alpha; } //! Modify the non zero gradient. double& Alpha() { return alpha; } - //! Serialize the layer. + //! Get size of weights. + size_t WeightSize() const { return 0; } + + /** + * Serialize the layer. + */ template void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Leakyness Parameter in the range 0 LeakyReLU; - +}; // class LeakyReLU } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp b/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp index 7a6e8dce3a..d03009f348 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu_impl.hpp @@ -20,68 +20,27 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -LeakyReLUType::LeakyReLUType(const double alpha) : - Layer(), - alpha(alpha) +template +LeakyReLU::LeakyReLU( + const double alpha) : alpha(alpha) { // Nothing to do here. } -template -LeakyReLUType::LeakyReLUType(const LeakyReLUType& other) : - Layer(other), - alpha(other.alpha) -{ - // Nothing to do. -} - -template -LeakyReLUType::LeakyReLUType( - LeakyReLUType&& other) : - Layer(std::move(other)), - alpha(std::move(other.alpha)) -{ - // Nothing to do. -} - -template -LeakyReLUType& -LeakyReLUType::operator=(const LeakyReLUType& other) -{ - if (&other != this) - { - Layer::operator=(other); - alpha = other.alpha; - } - - return *this; -} - -template -LeakyReLUType& -LeakyReLUType::operator=(LeakyReLUType&& other) -{ - if (&other != this) - { - Layer::operator=(std::move(other)); - alpha = std::move(other.alpha); - } - - return *this; -} - -template -void LeakyReLUType::Forward(const MatType& input, MatType& output) +template +template +void LeakyReLU::Forward( + const InputType& input, OutputType& output) { output = arma::max(input, alpha * input); } -template -void LeakyReLUType::Backward( - const MatType& input, const MatType& gy, MatType& g) +template +template +void LeakyReLU::Backward( + const DataType& input, const DataType& gy, DataType& g) { - MatType derivative; + DataType derivative; derivative.set_size(arma::size(input)); for (size_t i = 0; i < input.n_elem; ++i) derivative(i) = (input(i) >= 0) ? 1 : alpha; @@ -89,14 +48,12 @@ void LeakyReLUType::Backward( g = gy % derivative; } -template +template template -void LeakyReLUType::serialize( +void LeakyReLU::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(alpha)); } diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp index 7dca66ea01..5b9d23bd87 100644 --- a/src/mlpack/methods/ann/layer/linear.hpp +++ b/src/mlpack/methods/ann/layer/linear.hpp @@ -16,7 +16,7 @@ #include #include -#include "layer.hpp" +#include "layer_types.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -25,120 +25,142 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the Linear layer class. The Linear class represents a * single layer of a neural network. * - * The linear layer applies a linear transformation to the incoming data - * (input), i.e. y = Ax + b. The input matrix given in Forward(input, output) - * must be either a vector or matrix. If the input is a matrix, then each column - * is assumed to be an input sample of given batch. - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. - * @tparam RegularizerType Type of the regularizer to be used (Default no - * regularizer). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template< - typename MatType = arma::mat, +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, typename RegularizerType = NoRegularizer > -class LinearType : public Layer +class Linear { public: //! Create the Linear object. - LinearType(); + Linear(); /** - * Create the Linear layer object with the specified number of output - * dimensions. + * Create the Linear layer object using the specified number of units. * - * @param outSize The output dimension. - * @param regularizer The regularizer to use, optional (default: no - * regularizer). + * @param inSize The number of input units. + * @param outSize The number of output units. + * @param regularizer The regularizer to use, optional. */ - LinearType(const size_t outSize, - RegularizerType regularizer = RegularizerType()); + Linear(const size_t inSize, + const size_t outSize, + RegularizerType regularizer = RegularizerType()); - virtual ~LinearType() { } + //! Copy constructor. + Linear(const Linear& layer); - //! Clone the LinearType object. This handles polymorphism correctly. - LinearType* Clone() const { return new LinearType(*this); } + //! Move constructor. + Linear(Linear&&); - //! Copy the other Linear layer (but not weights). - LinearType(const LinearType& layer); + //! Copy assignment operator. + Linear& operator=(const Linear& layer); - //! Take ownership of the members of the other Linear layer (but not weights). - LinearType(LinearType&& layer); + //! Move assignment operator. + Linear& operator=(Linear&& layer); - //! Copy the other Linear layer (but not weights). - LinearType& operator=(const LinearType& layer); - - //! Take ownership of the members of the other Linear layer (but not weights). - LinearType& operator=(LinearType&& layer); - - /** - * Reset the layer parameter (weights and bias). The method is called to - * assign the allocated memory to the internal learnable parameters. + /* + * Reset the layer parameter. */ - void SetWeights(typename MatType::elem_type* weightsPtr); + void Reset(); /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. * - * f(x) is a linear transformation: Ax + b, where x is the given input, x are - * the layer weights and b is the layer bias. - * * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function * f(x) by propagating x backwards trough f. Using the results from the feed * forward pass. * - * To compute the downstream gradient (g) the chain rule is used. - * * @param * (input) The propagated input activation. * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, - const MatType& gy, - MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); - /** + /* * Calculate the gradient using the output delta and the input activation. * * @param input The input parameter used for calculating the gradient. * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const MatType& input, - const MatType& error, - MatType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - const MatType& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - MatType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the input size. + size_t InputSize() const { return inSize; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the weight of the layer. - MatType const& Weight() const { return weight; } + OutputDataType const& Weight() const { return weight; } //! Modify the weight of the layer. - MatType& Weight() { return weight; } + OutputDataType& Weight() { return weight; } //! Get the bias of the layer. - MatType const& Bias() const { return bias; } + OutputDataType const& Bias() const { return bias; } //! Modify the bias weights of the layer. - MatType& Bias() { return bias; } + OutputDataType& Bias() { return bias; } //! Get the size of the weights. - size_t WeightSize() const { return (inSize * outSize) + outSize; } + size_t WeightSize() const + { + return (inSize * outSize) + outSize; + } - //! Compute the output dimensions of the layer given `InputDimensions()`. - void ComputeOutputDimensions(); + //! Get the shape of the input. + size_t InputShape() const + { + return inSize; + } - //! Serialize the layer. + /** + * Serialize the layer + */ template void serialize(Archive& ar, const uint32_t /* version */); @@ -149,24 +171,30 @@ class LinearType : public Layer //! Locally-stored number of output units. size_t outSize; - //! Locally-stored weight object. This holds all the weights in a vectorized - //! form; i.e., the weights and the bias. - MatType weights; + //! Locally-stored weight object. + OutputDataType weights; //! Locally-stored weight parameters. - MatType weight; + OutputDataType weight; //! Locally-stored bias term parameters. - MatType bias; + OutputDataType bias; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; //! Locally-stored regularizer object. RegularizerType regularizer; -}; // class LinearType - -// Convenience typedefs. - -// Standard Linear layer using no regularization. -typedef LinearType Linear; +}; // class Linear } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/linear3d.hpp b/src/mlpack/methods/ann/layer/linear3d.hpp index 6b9b554ea7..24e3de56e6 100644 --- a/src/mlpack/methods/ann/layer/linear3d.hpp +++ b/src/mlpack/methods/ann/layer/linear3d.hpp @@ -14,10 +14,9 @@ #define MLPACK_METHODS_ANN_LAYER_LINEAR3D_HPP #include +#include #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -28,48 +27,49 @@ namespace ann /** Artificial Neural Network. */ { * Shape of input : (inSize * nPoints, batchSize) * Shape of output : (outSize * nPoints, batchSize) * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template< - typename MatType = arma::mat, +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, typename RegularizerType = NoRegularizer > -class Linear3DType : public Layer +class Linear3D { public: //! Create the Linear3D object. - Linear3DType(); + Linear3D(); /** - * Create the Linear3D layer object using the specified number of output - * units. + * Create the Linear3D layer object using the specified number of units. * + * @param inSize The number of input units. * @param outSize The number of output units. * @param regularizer The regularizer to use, optional. */ - Linear3DType(const size_t outSize, - RegularizerType regularizer = RegularizerType()); + Linear3D(const size_t inSize, + const size_t outSize, + RegularizerType regularizer = RegularizerType()); - //! Clone the Linear3DType object. This handles polymorphism correctly. - Linear3DType* Clone() const { return new Linear3DType(*this); } + //! Copy constructor. + Linear3D(const Linear3D& layer); - // Virtual destructor. - virtual ~Linear3DType() { } + //! Move constructor. + Linear3D(Linear3D&&); - //! Copy the given Linear3DType (but not weights). - Linear3DType(const Linear3DType& other); - //! Take ownership of the given Linear3DType (but not weights). - Linear3DType(Linear3DType&& other); - //! Copy the given Linear3DType (but not weights). - Linear3DType& operator=(const Linear3DType& other); - //! Take ownership of the given Linear3DType (but not weights). - Linear3DType& operator=(Linear3DType&& other); + //! Copy assignment operator. + Linear3D& operator=(const Linear3D& layer); + + //! Move assignment operator. + Linear3D& operator=(Linear3D&& layer); /* * Reset the layer parameter. */ - void SetWeights(typename MatType::elem_type* weightsPtr); + void Reset(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -78,7 +78,8 @@ class Linear3DType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -89,41 +90,69 @@ class Linear3DType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, - const MatType& gy, - MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); - /** + /* * Calculate the gradient using the output delta and the input activation. * * @param input The input parameter used for calculating the gradient. * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const MatType& input, - const MatType& error, - MatType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - MatType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - MatType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the input size. + size_t InputSize() const { return inSize; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the weight of the layer. - MatType const& Weight() const { return weight; } + OutputDataType const& Weight() const { return weight; } //! Modify the weight of the layer. - MatType& Weight() { return weight; } + OutputDataType& Weight() { return weight; } //! Get the bias of the layer. - MatType const& Bias() const { return bias; } + OutputDataType const& Bias() const { return bias; } //! Modify the bias weights of the layer. - MatType& Bias() { return bias; } + OutputDataType& Bias() { return bias; } - //! Return the number of weight elements. - size_t WeightSize() const { return outSize * (this->inputDimensions[0] + 1); } - - //! Compute the output dimensions for the layer, using `InputDimensions()`. - void ComputeOutputDimensions(); + //! Get the shape of the input. + size_t InputShape() const + { + return inSize; + } /** * Serialize the layer @@ -132,25 +161,37 @@ class Linear3DType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored number of input units. + size_t inSize; + //! Locally-stored number of output units. size_t outSize; //! Locally-stored weight object. - MatType weights; + OutputDataType weights; //! Locally-stored weight parameters. - MatType weight; + OutputDataType weight; //! Locally-stored bias term parameters. - MatType bias; + OutputDataType bias; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; //! Locally-stored regularizer object. RegularizerType regularizer; }; // class Linear -// Standard Linear3D layer. -typedef Linear3DType Linear3D; - } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/linear3d_impl.hpp b/src/mlpack/methods/ann/layer/linear3d_impl.hpp index cc4ae74375..37cc9e39e5 100644 --- a/src/mlpack/methods/ann/layer/linear3d_impl.hpp +++ b/src/mlpack/methods/ann/layer/linear3d_impl.hpp @@ -18,94 +18,116 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -Linear3DType::Linear3DType() : - Layer(), +template +Linear3D::Linear3D() : + inSize(0), outSize(0) { // Nothing to do here. } -template -Linear3DType::Linear3DType( +template +Linear3D::Linear3D( + const size_t inSize, const size_t outSize, RegularizerType regularizer) : - Layer(), + inSize(inSize), outSize(outSize), regularizer(regularizer) -{ } - -template -Linear3DType::Linear3DType( - const Linear3DType& other) : - Layer(other), - outSize(other.outSize), - regularizer(other.regularizer) { - // Nothing to do. + weights.set_size(outSize * inSize + outSize, 1); } -template -Linear3DType::Linear3DType( - Linear3DType&& other) : - Layer(std::move(other)), - outSize(std::move(other.outSize)), - regularizer(std::move(other.regularizer)) +template +Linear3D::Linear3D( + const Linear3D& layer) : + inSize(layer.inSize), + outSize(layer.outSize), + weights(layer.weights), + regularizer(layer.regularizer) { - // Nothing to do. + // Nothing to do here. } -template -Linear3DType& -Linear3DType::operator=( - const Linear3DType& other) +template +Linear3D::Linear3D( + Linear3D&& layer) : + inSize(0), + outSize(0), + weights(std::move(layer.weights)), + regularizer(std::move(layer.regularizer)) { - if (&other != this) + // Nothing to do here. +} + +template +Linear3D& +Linear3D:: +operator=(const Linear3D& layer) +{ + if (this != &layer) { - Layer::operator=(other); - outSize = other.outSize; - regularizer = other.regularizer; + inSize = layer.inSize; + outSize = layer.outSize; + weights = layer.weights; + regularizer = layer.regularizer; } - return *this; } -template -Linear3DType& -Linear3DType::operator=( - Linear3DType&& other) +template +Linear3D& +Linear3D:: +operator=(Linear3D&& layer) { - if (&other != this) + if (this != &layer) { - Layer::operator=(std::move(other)); - outSize = std::move(other.outSize); - regularizer = std::move(other.regularizer); + inSize = 0; + outSize = 0; + weights = std::move(layer.weights); + regularizer = std::move(layer.regularizer); } - return *this; } -template -void Linear3DType::SetWeights( - typename MatType::elem_type* weightsPtr) +template +void Linear3D::Reset() { - MakeAlias(weights, weightsPtr, outSize * this->inputDimensions[0] + outSize, - 1); - MakeAlias(weight, weightsPtr, outSize, this->inputDimensions[0]); - MakeAlias(bias, weightsPtr + weight.n_elem, outSize, 1); + typedef typename arma::Mat MatType; + + weight = MatType(weights.memptr(), outSize, inSize, false, false); + bias = MatType(weights.memptr() + weight.n_elem, outSize, 1, false, false); } -template -void Linear3DType::Forward( - const MatType& input, MatType& output) +template +template +void Linear3D::Forward( + const arma::Mat& input, arma::Mat& output) { - typedef typename arma::Cube CubeType; + typedef typename arma::Mat MatType; + typedef typename arma::Cube CubeType; - const size_t nPoints = input.n_rows / this->inputDimensions[0]; + if (input.n_rows % inSize != 0) + { + Log::Fatal << "Number of features in the input must be divisible by inSize." + << std::endl; + } + + const size_t nPoints = input.n_rows / inSize; const size_t batchSize = input.n_cols; - const CubeType inputTemp(const_cast(input).memptr(), - this->inputDimensions[0], nPoints, batchSize, false, false); + output.set_size(outSize * nPoints, batchSize); + + const CubeType inputTemp(const_cast(input).memptr(), inSize, + nPoints, batchSize, false, false); for (size_t i = 0; i < batchSize; ++i) { @@ -117,13 +139,16 @@ void Linear3DType::Forward( } } -template -void Linear3DType::Backward( - const MatType& /* input */, - const MatType& gy, - MatType& g) +template +template +void Linear3D::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - typedef typename arma::Cube CubeType; + typedef typename arma::Mat MatType; + typedef typename arma::Cube CubeType; if (gy.n_rows % outSize != 0) { @@ -137,6 +162,8 @@ void Linear3DType::Backward( const CubeType gyTemp(const_cast(gy).memptr(), outSize, nPoints, batchSize, false, false); + g.set_size(inSize * nPoints, batchSize); + for (size_t i = 0; i < gyTemp.n_slices; ++i) { // Shape of weight : (outSize, inSize). @@ -145,26 +172,29 @@ void Linear3DType::Backward( } } -template -void Linear3DType::Gradient( - const MatType& input, - const MatType& error, - MatType& gradient) +template +template +void Linear3D::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - typedef typename arma::Cube CubeType; + typedef typename arma::Mat MatType; + typedef typename arma::Cube CubeType; if (error.n_rows % outSize != 0) Log::Fatal << "Propagated error matrix has invalid dimension!" << std::endl; - const size_t nPoints = input.n_rows / this->inputDimensions[0]; + const size_t nPoints = input.n_rows / inSize; const size_t batchSize = input.n_cols; - const CubeType inputTemp(const_cast(input).memptr(), - this->inputDimensions[0], nPoints, batchSize, false, false); + const CubeType inputTemp(const_cast(input).memptr(), inSize, + nPoints, batchSize, false, false); const CubeType errorTemp(const_cast(error).memptr(), outSize, nPoints, batchSize, false, false); - CubeType dW(outSize, this->inputDimensions[0], batchSize); + CubeType dW(outSize, inSize, batchSize); for (size_t i = 0; i < batchSize; ++i) { // Shape of errorTemp : (outSize, nPoints, batchSize). @@ -172,6 +202,8 @@ void Linear3DType::Gradient( dW.slice(i) = errorTemp.slice(i) * inputTemp.slice(i).t(); } + gradient.set_size(arma::size(weights)); + gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise(arma::sum(dW, 2)); @@ -181,27 +213,19 @@ void Linear3DType::Gradient( regularizer.Evaluate(weights, gradient); } -template -void Linear3DType< - MatType, RegularizerType ->::ComputeOutputDimensions() -{ - // The Linear3D layer shares weights for each row of the input, and - // duplicates it across the columns. Thus, we only change the number of - // rows. - this->outputDimensions = this->inputDimensions; - this->outputDimensions[0] = outSize; -} - -template +template template -void Linear3DType::serialize( +void Linear3D::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - + ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(outSize)); - ar(CEREAL_NVP(regularizer)); + + // This is inefficient, but we have to allocate this memory so that + // WeightSetVisitor gets the right size. + if (cereal::is_loading()) + weights.set_size(outSize * inSize + outSize, 1); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/linear_impl.hpp b/src/mlpack/methods/ann/layer/linear_impl.hpp index fe7841e791..2edfb4802c 100644 --- a/src/mlpack/methods/ann/layer/linear_impl.hpp +++ b/src/mlpack/methods/ann/layer/linear_impl.hpp @@ -19,141 +19,136 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -LinearType::LinearType() : - Layer(), +template +Linear::Linear() : inSize(0), outSize(0) { // Nothing to do here. } -template -LinearType::LinearType( +template +Linear::Linear( + const size_t inSize, const size_t outSize, RegularizerType regularizer) : - Layer(), - inSize(0), // This will be computed in ComputeOutputDimensions(). + inSize(inSize), outSize(outSize), regularizer(regularizer) { weights.set_size(WeightSize(), 1); } -// Copy constructor. -template -LinearType::LinearType(const LinearType& layer) : - Layer(layer), +template +Linear::Linear( + const Linear& layer) : inSize(layer.inSize), outSize(layer.outSize), + weights(layer.weights), regularizer(layer.regularizer) { - // Nothing else to do. + // Nothing to do here. } -// Move constructor. -template -LinearType::LinearType(LinearType&& layer) : - Layer(std::move(layer)), - inSize(std::move(layer.inSize)), - outSize(std::move(layer.outSize)), +template +Linear::Linear( + Linear&& layer) : + inSize(0), + outSize(0), + weights(std::move(layer.weights)), regularizer(std::move(layer.regularizer)) { - // Nothing else to do. + // Nothing to do here. } -template -LinearType& -LinearType::operator=(const LinearType& layer) +template +Linear& +Linear:: +operator=(const Linear& layer) { - if (&layer != this) + if (this != &layer) { - Layer::operator=(layer); inSize = layer.inSize; outSize = layer.outSize; + weights = layer.weights; regularizer = layer.regularizer; } - return *this; } -template -LinearType& -LinearType::operator=( - LinearType&& layer) +template +Linear& +Linear:: +operator=(Linear&& layer) { - if (&layer != this) + if (this != &layer) { - Layer::operator=(std::move(layer)); - inSize = std::move(layer.inSize); - outSize = std::move(layer.outSize); + inSize = layer.inSize; + outSize = layer.outSize; + weights = std::move(layer.weights); regularizer = std::move(layer.regularizer); } - return *this; } -template -void LinearType::SetWeights( - typename MatType::elem_type* weightsPtr) +template +void Linear::Reset() { - MakeAlias(weights, weightsPtr, outSize * inSize + outSize, 1); - MakeAlias(weight, weightsPtr, outSize, inSize); - MakeAlias(bias, weightsPtr + weight.n_elem, outSize, 1); + weight = arma::mat(weights.memptr(), outSize, inSize, false, false); + bias = arma::mat(weights.memptr() + weight.n_elem, + outSize, 1, false, false); } -template -void LinearType::Forward( - const MatType& input, MatType& output) +template +template +void Linear::Forward( + const arma::Mat& input, arma::Mat& output) { output = weight * input; output.each_col() += bias; } -template -void LinearType::Backward( - const MatType& /* input */, const MatType& gy, MatType& g) +template +template +void Linear::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { g = weight.t() * gy; } -template -void LinearType::Gradient( - const MatType& input, - const MatType& error, - MatType& gradient) +template +template +void Linear::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise( error * input.t()); gradient.submat(weight.n_elem, 0, gradient.n_elem - 1, 0) = arma::sum(error, 1); - regularizer.Evaluate(weights, gradient); } -template -void LinearType::ComputeOutputDimensions() -{ - inSize = this->inputDimensions[0]; - for (size_t i = 1; i < this->inputDimensions.size(); ++i) - inSize *= this->inputDimensions[i]; - this->outputDimensions = std::vector(this->inputDimensions.size(), - 1); - - // The Linear layer flattens its input. - this->outputDimensions[0] = outSize; -} - -template +template template -void LinearType::serialize( +void Linear::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(outSize)); - ar(CEREAL_NVP(regularizer)); + ar(CEREAL_NVP(weights)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/linear_no_bias.hpp b/src/mlpack/methods/ann/layer/linear_no_bias.hpp index 00bf074d7a..1e85933135 100644 --- a/src/mlpack/methods/ann/layer/linear_no_bias.hpp +++ b/src/mlpack/methods/ann/layer/linear_no_bias.hpp @@ -16,7 +16,7 @@ #include #include -#include "layer.hpp" +#include "layer_types.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -25,50 +25,48 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the LinearNoBias class. The LinearNoBias class represents a * single layer of a neural network. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. - * @tparam RegularizerType Type of the regularizer to be used (Default no - * regularizer). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template< - typename MatType = arma::mat, +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, typename RegularizerType = NoRegularizer > -class LinearNoBiasType : public Layer +class LinearNoBias { public: //! Create the LinearNoBias object. - LinearNoBiasType(); - + LinearNoBias(); /** * Create the LinearNoBias object using the specified number of units. * + * @param inSize The number of input units. * @param outSize The number of output units. * @param regularizer The regularizer to use, optional. */ - LinearNoBiasType(const size_t outSize, - RegularizerType regularizer = RegularizerType()); - - //! Clone the LinearNoBiasType object. This handles polymorphism correctly. - LinearNoBiasType* Clone() const { return new LinearNoBiasType(*this); } - - //! Reset the layer parameter. - void SetWeights(typename MatType::elem_type* weightsPtr); + LinearNoBias(const size_t inSize, + const size_t outSize, + RegularizerType regularizer = RegularizerType()); //! Copy constructor. - LinearNoBiasType(const LinearNoBiasType& layer); + LinearNoBias(const LinearNoBias& layer); //! Move constructor. - LinearNoBiasType(LinearNoBiasType&&); + LinearNoBias(LinearNoBias&&); //! Copy assignment operator. - LinearNoBiasType& operator=(const LinearNoBiasType& layer); + LinearNoBias& operator=(const LinearNoBias& layer); //! Move assignment operator. - LinearNoBiasType& operator=(LinearNoBiasType&& layer); + LinearNoBias& operator=(LinearNoBias&& layer); - //! Virtual destructor. - virtual ~LinearNoBiasType() { } + /* + * Reset the layer parameter. + */ + void Reset(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -77,7 +75,8 @@ class LinearNoBiasType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -88,33 +87,69 @@ class LinearNoBiasType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, - const MatType& gy, - MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); - /** + /* * Calculate the gradient using the output delta and the input activation. * * @param input The input parameter used for calculating the gradient. * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const MatType& input, - const MatType& error, - MatType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - const MatType& Parameters() const { return weight; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - MatType& Parameters() { return weight; } + OutputDataType& Parameters() { return weights; } - //! Get the number of weights in the layer. - size_t WeightSize() const { return inSize * outSize; } + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } - //! Compute the output dimensions of the layer using `InputDimensions()`. - void ComputeOutputDimensions(); + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } - //! Serialize the layer. + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the input size. + size_t InputSize() const { return inSize; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the size of the weights. + size_t WeightSize() const + { + return inSize * outSize; + } + + //! Get the shape of the input. + size_t InputShape() const + { + return inSize; + } + + /** + * Serialize the layer + */ template void serialize(Archive& ar, const uint32_t /* version */); @@ -125,17 +160,27 @@ class LinearNoBiasType : public Layer //! Locally-stored number of output units. size_t outSize; + //! Locally-stored weight object. + OutputDataType weights; + //! Locally-stored weight parameter. - MatType weight; + OutputDataType weight; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; //! Locally-stored regularizer object. RegularizerType regularizer; -}; // class LinearNoBiasType - -// Convenience typedefs. - -// Standard Linear without bias layer using no regularization. -typedef LinearNoBiasType LinearNoBias; +}; // class LinearNoBias } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp b/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp index bc2f18702c..ed26cd59fa 100644 --- a/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp +++ b/src/mlpack/methods/ann/layer/linear_no_bias_impl.hpp @@ -19,136 +19,135 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -LinearNoBiasType::LinearNoBiasType() : - Layer(), +template +LinearNoBias::LinearNoBias() : inSize(0), outSize(0) { // Nothing to do here. } -template -LinearNoBiasType::LinearNoBiasType( +template +LinearNoBias::LinearNoBias( + const size_t inSize, const size_t outSize, RegularizerType regularizer) : - Layer(), - inSize(0), // This will be set by ComputeOutputDimensions(). + inSize(inSize), outSize(outSize), regularizer(regularizer) { - // Nothing to do. + weights.set_size(WeightSize(), 1); } -template -LinearNoBiasType::LinearNoBiasType( - const LinearNoBiasType& layer) : - Layer(layer), +template +LinearNoBias::LinearNoBias( + const LinearNoBias& layer) : inSize(layer.inSize), outSize(layer.outSize), + weights(layer.weights), regularizer(layer.regularizer) { // Nothing to do here. } -template -LinearNoBiasType::LinearNoBiasType( - LinearNoBiasType&& layer) : - Layer(std::move(layer)), +template +LinearNoBias::LinearNoBias( + LinearNoBias&& layer) : inSize(0), outSize(0), + weights(std::move(layer.weights)), regularizer(std::move(layer.regularizer)) { // Nothing to do here. } -template -LinearNoBiasType& -LinearNoBiasType::operator=( - const LinearNoBiasType& layer) +template +LinearNoBias& +LinearNoBias:: +operator=(const LinearNoBias& layer) { if (this != &layer) { - Layer::operator=(layer); inSize = layer.inSize; outSize = layer.outSize; + weights = layer.weights; regularizer = layer.regularizer; } - return *this; } -template -LinearNoBiasType& -LinearNoBiasType::operator=( - LinearNoBiasType&& layer) +template +LinearNoBias& +LinearNoBias:: +operator=(LinearNoBias&& layer) { if (this != &layer) { - Layer::operator=(std::move(layer)); - inSize = std::move(layer.inSize); - outSize = std::move(layer.outSize); + inSize = layer.inSize; + outSize = layer.outSize; + weights = std::move(layer.weights); regularizer = std::move(layer.regularizer); } - return *this; } -template -void LinearNoBiasType::SetWeights( - typename MatType::elem_type* weightsPtr) +template +void LinearNoBias::Reset() { - MakeAlias(weight, weightsPtr, outSize, inSize); + weight = arma::mat(weights.memptr(), outSize, inSize, false, false); } -template -void LinearNoBiasType::Forward( - const MatType& input, MatType& output) +template +template +void LinearNoBias::Forward( + const arma::Mat& input, arma::Mat& output) { output = weight * input; } -template -void LinearNoBiasType::Backward( - const MatType& /* input */, const MatType& gy, MatType& g) +template +template +void LinearNoBias::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { g = weight.t() * gy; } -template -void LinearNoBiasType::Gradient( - const MatType& input, - const MatType& error, - MatType& gradient) +template +template +void LinearNoBias::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise( error * input.t()); - regularizer.Evaluate(weight, gradient); + regularizer.Evaluate(weights, gradient); } -template -void LinearNoBiasType::ComputeOutputDimensions() -{ - inSize = this->inputDimensions[0]; - for (size_t i = 1; i < this->inputDimensions.size(); ++i) - inSize *= this->inputDimensions[i]; - - this->outputDimensions = std::vector(this->inputDimensions.size(), - 1); - - this->outputDimensions[0] = outSize; -} - -template +template template -void LinearNoBiasType::serialize( +void LinearNoBias::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(outSize)); - ar(CEREAL_NVP(regularizer)); + + // This is inefficient, but necessary so that WeightSetVisitor sets the right + // size. + if (cereal::is_loading()) + weights.set_size(outSize * inSize, 1); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/log_softmax.hpp b/src/mlpack/methods/ann/layer/log_softmax.hpp index a93edb6183..8c62fd2d9e 100644 --- a/src/mlpack/methods/ann/layer/log_softmax.hpp +++ b/src/mlpack/methods/ann/layer/log_softmax.hpp @@ -14,8 +14,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -26,32 +24,22 @@ namespace ann /** Artificial Neural Network. */ { * (NegativeLogLikelihoodLayer), which expects that the input contains * log-probabilities for each class. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class LogSoftMaxType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class LogSoftMax { public: /** - * Create the LogSoftmax layer. + * Create the LogSoftmax object. */ - LogSoftMaxType(); - - //! Clone the LogSoftMaxType object. This handles polymorphism correctly. - LogSoftMaxType* Clone() const { return new LogSoftMaxType(*this); } - - // Virtual destructor. - virtual ~LogSoftMaxType() { } - - //! Copy the given LogSoftMaxType. - LogSoftMaxType(const LogSoftMaxType& other); - //! Take ownership of the given LogSoftMaxType. - LogSoftMaxType(LogSoftMaxType&& other); - //! Copy the given LogSoftMaxType. - LogSoftMaxType& operator=(const LogSoftMaxType& other); - //! Take ownership of the given LogSoftMaxType. - LogSoftMaxType& operator=(LogSoftMaxType&& other); + LogSoftMax(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -60,7 +48,8 @@ class LogSoftMaxType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const InputType& input, OutputType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -71,22 +60,34 @@ class LogSoftMaxType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& input, const MatType& gy, MatType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + InputDataType& Delta() const { return delta; } + //! Modify the delta. + InputDataType& Delta() { return delta; } + + /** + * Serialize the layer. + */ template - void serialize(Archive& ar, const uint32_t /* version */) - { - ar(cereal::base_class>(this)); - // Nothing to do. - } + void serialize(Archive& /* ar */, const uint32_t /* version */); private: -}; // class LogSoftmaxType + //! Locally-stored delta object. + OutputDataType delta; -// Convenience typedefs. - -// Standard Linear layer using no regularization. -typedef LogSoftMaxType LogSoftMax; + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class LogSoftmax } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/log_softmax_impl.hpp b/src/mlpack/methods/ann/layer/log_softmax_impl.hpp index 37b2e296bd..4762662c2f 100644 --- a/src/mlpack/methods/ann/layer/log_softmax_impl.hpp +++ b/src/mlpack/methods/ann/layer/log_softmax_impl.hpp @@ -18,54 +18,18 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -LogSoftMaxType::LogSoftMaxType() +template +LogSoftMax::LogSoftMax() { // Nothing to do here. } -template -LogSoftMaxType::LogSoftMaxType(const LogSoftMaxType& other) : - Layer(other) +template +template +void LogSoftMax::Forward( + const InputType& input, OutputType& output) { - // Nothing to do here. -} - -template -LogSoftMaxType::LogSoftMaxType(LogSoftMaxType&& other) : - Layer(std::move(other)) -{ - // Nothing to do here. -} - -template -LogSoftMaxType& -LogSoftMaxType::operator=(const LogSoftMaxType& other) -{ - if (&other != this) - { - Layer::operator=(other); - } - - return *this; -} - -template -LogSoftMaxType& -LogSoftMaxType::operator=(LogSoftMaxType&& other) -{ - if (&other != this) - { - Layer::operator=(std::move(other)); - } - - return *this; -} - -template -void LogSoftMaxType::Forward(const MatType& input, MatType& output) -{ - MatType maxInput = arma::repmat(arma::max(input), input.n_rows, 1); + arma::mat maxInput = arma::repmat(arma::max(input), input.n_rows, 1); output = (maxInput - input); // Approximation of the base-e exponential function. The acuracy however is @@ -97,15 +61,25 @@ void LogSoftMaxType::Forward(const MatType& input, MatType& output) output = input - maxInput; } -template -void LogSoftMaxType::Backward( - const MatType& input, - const MatType& gy, - MatType& g) +template +template +void LogSoftMax::Backward( + const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { g = arma::exp(input) + gy; } +template +template +void LogSoftMax::serialize( + Archive& /* ar */, + const uint32_t /* version */) +{ + // Nothing to do here. +} + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/lookup.hpp b/src/mlpack/methods/ann/layer/lookup.hpp similarity index 64% rename from src/mlpack/methods/ann/layer/not_adapted/lookup.hpp rename to src/mlpack/methods/ann/layer/lookup.hpp index 714924949d..6f6796f193 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/lookup.hpp +++ b/src/mlpack/methods/ann/layer/lookup.hpp @@ -14,7 +14,7 @@ #define MLPACK_METHODS_ANN_LAYER_LOOKUP_HPP #include -#include "layer.hpp" +#include namespace mlpack { namespace ann /* Artificial Neural Network. */ { @@ -29,16 +29,16 @@ namespace ann /* Artificial Neural Network. */ { * The input shape : (sequenceLength, batchSize). * The output shape : (embeddingSize, sequenceLength, batchSize). * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class LookupType : public Layer +class Lookup { public: /** @@ -47,10 +47,7 @@ class LookupType : public Layer * @param vocabSize The size of the vocabulary. * @param embeddingSize The length of each embedding vector. */ - LookupType(const size_t vocabSize = 0, const size_t embeddingSize = 0); - - //! Clone the LookupType object. This handles polymorphism correctly. - LookupType* Clone() const { return new LookupType(*this); } + Lookup(const size_t vocabSize = 0, const size_t embeddingSize = 0); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -59,7 +56,8 @@ class LookupType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -70,9 +68,10 @@ class LookupType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activation. @@ -81,14 +80,30 @@ class LookupType : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the size of the vocabulary. size_t VocabSize() const { return vocabSize; } @@ -96,19 +111,6 @@ class LookupType : public Layer //! Get the length of each embedding vector. size_t EmbeddingSize() const { return embeddingSize; } - //! Get the number of trainable parameters. - const size_t WeightSize() const { return embeddingSize * vocabSize; } - - //! Get the dimensions of the output. This layer adds an extra dimension for - //! the embedding. - const std::vector& OutputDimensions() const - { - std::vector result(inputDimensions.size() + 1, embeddingSize); - for (size_t i = 0; i < inputDimensions.size(); ++i) - result[i + 1] = inputDimensions[i]; - return result; - } - /** * Serialize the layer */ @@ -123,14 +125,21 @@ class LookupType : public Layer size_t embeddingSize; //! Locally-stored weight object. - OutputType weights; + OutputDataType weights; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; }; // class Lookup // Alias for using as embedding layer. -// template -// using Embedding = Lookup; -typedef LookupType Lookup; -typedef LookupType Embedding; +template +using Embedding = Lookup; } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/lookup_impl.hpp b/src/mlpack/methods/ann/layer/lookup_impl.hpp similarity index 58% rename from src/mlpack/methods/ann/layer/not_adapted/lookup_impl.hpp rename to src/mlpack/methods/ann/layer/lookup_impl.hpp index d1e36d2f29..7755f83342 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/lookup_impl.hpp +++ b/src/mlpack/methods/ann/layer/lookup_impl.hpp @@ -19,30 +19,26 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -LookupType::LookupType( +template +Lookup::Lookup( const size_t vocabSize, const size_t embeddingSize) : vocabSize(vocabSize), embeddingSize(embeddingSize) { - // Nothing to do. + weights.set_size(embeddingSize, vocabSize); } -template -void LookupType::SetWeights( - typename OutputType::elem_type* weightsPtr) -{ - weights = OutputType(weightsPtr, embeddingSize, vocabSize, false, true); -} - -template -void LookupType::Forward( - const InputType& input, OutputType& output) +template +template +void Lookup::Forward( + const arma::Mat& input, arma::Mat& output) { const size_t seqLength = input.n_rows; const size_t batchSize = input.n_cols; + output.set_size(embeddingSize * seqLength, batchSize); + for (size_t i = 0; i < batchSize; ++i) { // ith column of output is a vectorized form of a matrix of shape @@ -53,29 +49,32 @@ void LookupType::Forward( } } -template -void LookupType::Backward( - const InputType& /* input */, - const OutputType& /* gy */, - OutputType& /* g */) +template +template +void Lookup::Backward( + const arma::Mat& /* input */, + const arma::Mat& /* gy */, + arma::Mat& /* g */) { Log::Fatal << "Lookup cannot be used as an intermediate layer." << std::endl; } -template -void LookupType::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) +template +template +void Lookup::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - typedef typename arma::Cube CubeType; const size_t seqLength = input.n_rows; const size_t batchSize = input.n_cols; - const CubeType errorTemp(const_cast(error).memptr(), + arma::Cube errorTemp(const_cast&>(error).memptr(), embeddingSize, seqLength, batchSize, false, false); + gradient.set_size(arma::size(weights)); gradient.zeros(); + for (size_t i = 0; i < batchSize; ++i) { gradient.cols(arma::conv_to::from(input.col(i)) - 1) @@ -83,15 +82,18 @@ void LookupType::Gradient( } } -template +template template -void LookupType::serialize( +void Lookup::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(vocabSize)); ar(CEREAL_NVP(embeddingSize)); + + // This is inefficient, but we have to allocate this memory so that + // WeightSetVisitor gets the right size. + if (cereal::is_loading()) + weights.set_size(embeddingSize, vocabSize); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/lp_pooling.hpp b/src/mlpack/methods/ann/layer/lp_pooling.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/lp_pooling.hpp rename to src/mlpack/methods/ann/layer/lp_pooling.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/lp_pooling_impl.hpp b/src/mlpack/methods/ann/layer/lp_pooling_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/lp_pooling_impl.hpp rename to src/mlpack/methods/ann/layer/lp_pooling_impl.hpp diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index e2e297b2f2..effca7328e 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -15,8 +15,6 @@ #include #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -52,43 +50,43 @@ namespace ann /** Artificial Neural Network. */ { * \see FastLSTM for a faster LSTM version which combines the calculation of the * input, forget, output gates and hidden state in a single step. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class LSTMType : public RecurrentLayer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class LSTM { public: //! Create the LSTM object. - LSTMType(); + LSTM(); /** * Create the LSTM layer object using the specified parameters. * + * @param inSize The number of input units. * @param outSize The number of output units. * @param rho Maximum number of steps to backpropagate through time (BPTT). */ - LSTMType(const size_t outSize); + LSTM(const size_t inSize, + const size_t outSize, + const size_t rho = std::numeric_limits::max()); - //! Clone the LSTMType object. This handles polymorphism correctly. - LSTMType* Clone() const { return new LSTMType(*this); } + //! Copy constructor. + LSTM(const LSTM& layer); - //! Copy the given LSTMType object. - LSTMType(const LSTMType& other); - //! Take ownership of the given LSTMType object's data. - LSTMType(LSTMType&& other); - //! Copy the given LSTMType object. - LSTMType& operator=(const LSTMType& other); - //! Take ownership of the given LSTMType object's data. - LSTMType& operator=(LSTMType&& other); + //! Move constructor. + LSTM(LSTM&&); - virtual ~LSTMType() { } + //! Copy assignment operator. + LSTM& operator=(const LSTM& layer); - /** - * Reset the layer parameter. The method is called to - * assign the allocated memory to the internal learnable parameters. - */ - void SetWeights(typename MatType::elem_type* weightsPtr); + //! Move assignment operator. + LSTM& operator=(LSTM&& layer); /** * Ordinary feed-forward pass of a neural network, evaluating the function @@ -97,7 +95,23 @@ class LSTMType : public RecurrentLayer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const InputType& input, OutputType& output); + + /** + * Ordinary feed-forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + * @param cellState Cell state of the LSTM. + * @param useCellState Use the cellState passed in the LSTM cell. + */ + template + void Forward(const InputType& input, + OutputType& output, + OutputType& cellState, + bool useCellState = false); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -108,7 +122,23 @@ class LSTMType : public RecurrentLayer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& input, const MatType& gy, MatType& g); + template + void Backward(const InputType& input, + const ErrorType& gy, + GradientType& g); + + /* + * Reset the layer parameter. + */ + void Reset(); + + /* + * Resets the cell to accept a new input. This breaks the BPTT chain starts a + * new one. + * + * @param size The current maximum number of steps through time. + */ + void ResetCell(const size_t size); /* * Calculate the gradient using the output delta and the input activation. @@ -117,44 +147,56 @@ class LSTMType : public RecurrentLayer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const MatType& input, - const MatType& error, - MatType& gradient); + template + void Gradient(const InputType& input, + const ErrorType& error, + GradientType& gradient); - /** - * Reset the recurrent state of the LSTM layer, and allocate enough space to - * hold `bpttSteps` of previous passes with a batch size of `batchSize`. - * - * @param bpttSteps Number of steps of history to allocate space for. - * @param batchSize Batch size to prepare for. - */ - void ClearRecurrentState(const size_t bpttSteps, const size_t batchSize); + //! Get the maximum number of steps to backpropagate through time (BPTT). + size_t Rho() const { return rho; } + //! Modify the maximum number of steps to backpropagate through time (BPTT). + size_t& Rho() { return rho; } //! Get the parameters. - const MatType& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - MatType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } - //! Get the total number of trainable parameters. + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return grad; } + //! Modify the gradient. + OutputDataType& Gradient() { return grad; } + + //! Get the number of input units. + size_t InSize() const { return inSize; } + + //! Get the number of output units. + size_t OutSize() const { return outSize; } + + //! Get the size of the weights. size_t WeightSize() const { return (4 * outSize * inSize + 7 * outSize + 4 * outSize * outSize); } - //! Given a properly set InputDimensions(), compute the output dimensions. - void ComputeOutputDimensions() + //! Get the shape of the input. + size_t InputShape() const { - inSize = std::accumulate(this->inputDimensions.begin(), - this->inputDimensions.end(), 0); - this->outputDimensions = std::vector(this->inputDimensions.size(), - 1); - - // The LSTM layer flattens its input. - this->outputDimensions[0] = outSize; + return inSize; } /** - * Serialize the layer. + * Serialize the layer */ template void serialize(Archive& ar, const uint32_t /* version */); @@ -166,111 +208,148 @@ class LSTMType : public RecurrentLayer //! Locally-stored number of output units. size_t outSize; + //! Number of steps to backpropagate through time (BPTT). + size_t rho; + + //! Locally-stored number of forward steps. + size_t forwardStep; + + //! Locally-stored number of backward steps. + size_t backwardStep; + + //! Locally-stored number of gradient steps. + size_t gradientStep; + //! Locally-stored weight object. - MatType weights; + OutputDataType weights; - //! Weights between the output and input gate. - MatType output2GateInputWeight; + //! Locally-stored previous output. + OutputDataType prevOutput; - //! Weights between the input and gate. - MatType input2GateInputWeight; + //! Locally-stored batch size. + size_t batchSize; - //! Bias between the input and input gate. - MatType input2GateInputBias; + //! Current batch step, alias for batchSize - 1. + size_t batchStep; - //! Weights between the cell and input gate. - MatType cell2GateInputWeight; - - //! Weights between the output and forget gate. - MatType output2GateForgetWeight; - - //! Weights between the input and gate. - MatType input2GateForgetWeight; - - //! Bias between the input and gate. - MatType input2GateForgetBias; - - //! Bias between the input and gate. - MatType cell2GateForgetWeight; - - //! Weights between the output and gate. - MatType output2GateOutputWeight; - - //! Weights between the input and gate. - MatType input2GateOutputWeight; - - //! Bias between the input and gate. - MatType input2GateOutputBias; - - //! Weights between cell and output gate. - MatType cell2GateOutputWeight; - - // Below here are recurrent state matrices. - - //! Locally-stored input gate parameter. - MatType inputGate; - - //! Locally-stored forget gate parameter. - MatType forgetGate; - - //! Locally-stored hidden layer parameter. - MatType hiddenLayer; - - //! Locally-stored output gate parameter. - MatType outputGate; - - //! Locally-stored input to hidden weight. - MatType input2HiddenWeight; - - //! Locally-stored input to hidden bias. - MatType input2HiddenBias; - - //! Locally-stored output to hidden weight. - MatType output2HiddenWeight; - - //! Locally-stored cell parameter. - arma::Cube cell; - - // These members store recurrent state. - - //! Locally-stored input gate activation. - arma::Cube inputGateActivation; - - //! Locally-stored forget gate activation. - arma::Cube forgetGateActivation; - - //! Locally-stored output gate activation. - arma::Cube outputGateActivation; - - //! Locally-stored hidden layer activation. - arma::Cube hiddenLayerActivation; + //! Current gradient step to keep track of the backpropagate through time + //! step. + size_t gradientStepIdx; //! Locally-stored cell activation error. - arma::Cube cellActivation; + OutputDataType cellActivationError; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType grad; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Weights between the output and input gate. + OutputDataType output2GateInputWeight; + + //! Weights between the input and gate. + OutputDataType input2GateInputWeight; + + //! Bias between the input and input gate. + OutputDataType input2GateInputBias; + + //! Weights between the cell and input gate. + OutputDataType cell2GateInputWeight; + + //! Weights between the output and forget gate. + OutputDataType output2GateForgetWeight; + + //! Weights between the input and gate. + OutputDataType input2GateForgetWeight; + + //! Bias between the input and gate. + OutputDataType input2GateForgetBias; + + //! Bias between the input and gate. + OutputDataType cell2GateForgetWeight; + + //! Weights between the output and gate. + OutputDataType output2GateOutputWeight; + + //! Weights between the input and gate. + OutputDataType input2GateOutputWeight; + + //! Bias between the input and gate. + OutputDataType input2GateOutputBias; + + //! Weights between cell and output gate. + OutputDataType cell2GateOutputWeight; + + //! Locally-stored input gate parameter. + OutputDataType inputGate; + + //! Locally-stored forget gate parameter. + OutputDataType forgetGate; + + //! Locally-stored hidden layer parameter. + OutputDataType hiddenLayer; + + //! Locally-stored output gate parameter. + OutputDataType outputGate; + + //! Locally-stored input gate activation. + OutputDataType inputGateActivation; + + //! Locally-stored forget gate activation. + OutputDataType forgetGateActivation; + + //! Locally-stored output gate activation. + OutputDataType outputGateActivation; + + //! Locally-stored hidden layer activation. + OutputDataType hiddenLayerActivation; + + //! Locally-stored input to hidden weight. + OutputDataType input2HiddenWeight; + + //! Locally-stored input to hidden bias. + OutputDataType input2HiddenBias; + + //! Locally-stored output to hidden weight. + OutputDataType output2HiddenWeight; + + //! Locally-stored cell parameter. + OutputDataType cell; + + //! Locally-stored cell activation error. + OutputDataType cellActivation; //! Locally-stored forget gate error. - MatType forgetGateError; + OutputDataType forgetGateError; //! Locally-stored output gate error. - MatType outputGateError; + OutputDataType outputGateError; + + //! Locally-stored previous error. + OutputDataType prevError; //! Locally-stored output parameters. - arma::Cube outParameter; + OutputDataType outParameter; //! Locally-stored input cell error parameter. - MatType inputCellError; + OutputDataType inputCellError; //! Locally-stored input gate error. - MatType inputGateError; + OutputDataType inputGateError; //! Locally-stored hidden layer error. - MatType hiddenError; -}; // class LSTMType + OutputDataType hiddenError; -// Convenience typedefs. + //! Locally-stored current rho size. + size_t rhoSize; -// Standard LSTM layer. -typedef LSTMType LSTM; + //! Current backpropagate through time steps. + size_t bpttSteps; +}; // class LSTM } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index 078bcb1a1b..2b298d18aa 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -18,296 +18,430 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -LSTMType::LSTMType() : - RecurrentLayer(), - outSize(0) +template +LSTM::LSTM() { // Nothing to do here. } -template -LSTMType::LSTMType(const size_t outSize) : - RecurrentLayer(), - outSize(outSize) +template +LSTM::LSTM( + const LSTM& layer) : + inSize(layer.inSize), + outSize(layer.outSize), + rho(layer.rho), + forwardStep(layer.forwardStep), + backwardStep(layer.backwardStep), + gradientStep(layer.gradientStep), + weights(layer.weights), + batchSize(layer.batchSize), + batchStep(layer.batchStep), + gradientStepIdx(layer.gradientStepIdx), + rhoSize(layer.rho), + bpttSteps(layer.bpttSteps) { // Nothing to do here. } -template -LSTMType::LSTMType(const LSTMType& layer) : - RecurrentLayer(layer) +template +LSTM::LSTM( + LSTM&& layer) : + inSize(std::move(layer.inSize)), + outSize(std::move(layer.outSize)), + rho(std::move(layer.rho)), + forwardStep(std::move(layer.forwardStep)), + backwardStep(std::move(layer.backwardStep)), + gradientStep(std::move(layer.gradientStep)), + weights(std::move(layer.weights)), + batchSize(std::move(layer.batchSize)), + batchStep(std::move(layer.batchStep)), + gradientStepIdx(std::move(layer.gradientStepIdx)), + rhoSize(std::move(layer.rho)), + bpttSteps(std::move(layer.bpttSteps)) { // Nothing to do here. } -template -LSTMType::LSTMType(LSTMType&& layer) : - RecurrentLayer(std::move(layer)) -{ - // Nothing to do here. -} - -template -LSTMType& LSTMType::operator=(const LSTMType& layer) +template +LSTM& +LSTM :: operator=(const LSTM& layer) { if (this != &layer) { - RecurrentLayer::operator=(layer); + inSize = layer.inSize; + outSize = layer.outSize; + rho = layer.rho; + forwardStep = layer.forwardStep; + backwardStep = layer.backwardStep; + gradientStep = layer.gradientStep; + weights = layer.weights; + batchSize = layer.batchSize; + batchStep = layer.batchStep; + gradientStepIdx = layer.gradientStepIdx; + grad = layer.grad; + rhoSize = layer.rho; + bpttSteps = layer.bpttSteps; } - return *this; } -template -LSTMType& LSTMType::operator=(LSTMType&& layer) +template +LSTM& +LSTM :: operator=(LSTM&& layer) { if (this != &layer) { - RecurrentLayer::operator=(std::move(layer)); + inSize = std::move(layer.inSize); + outSize = std::move(layer.outSize); + rho = std::move(layer.rho); + forwardStep = std::move(layer.forwardStep); + backwardStep = std::move(layer.backwardStep); + gradientStep = std::move(layer.gradientStep); + weights = std::move(layer.weights); + batchSize = std::move(layer.batchSize); + batchStep = std::move(layer.batchStep); + gradientStepIdx = std::move(layer.gradientStepIdx); + grad = std::move(layer.grad); + rhoSize = std::move(layer.rho); + bpttSteps = std::move(layer.bpttSteps); } - return *this; } -template -void LSTMType::ClearRecurrentState( - const size_t bpttSteps, const size_t batchSize) +template +LSTM::LSTM( + const size_t inSize, const size_t outSize, const size_t rho) : + inSize(inSize), + outSize(outSize), + rho(rho), + forwardStep(0), + backwardStep(0), + gradientStep(0), + batchSize(0), + batchStep(0), + gradientStepIdx(0), + rhoSize(rho), + bpttSteps(0) { + weights.set_size(WeightSize(), 1); +} + +template +void LSTM::ResetCell(const size_t size) +{ + if (size == std::numeric_limits::max()) + return; + + rhoSize = size; + + if (batchSize == 0) + return; + + bpttSteps = std::min(rho, rhoSize); + forwardStep = 0; + gradientStepIdx = 0; + backwardStep = batchSize * size - 1; + gradientStep = batchSize * size - 1; + + const size_t rhoBatchSize = size * batchSize; + // Make sure all of the different matrices we will use to hold parameters are // at least as large as we need. - inputGate.set_size(outSize, batchSize); - forgetGate.set_size(outSize, batchSize); - hiddenLayer.set_size(outSize, batchSize); - outputGate.set_size(outSize, batchSize); + inputGate.set_size(outSize, rhoBatchSize); + forgetGate.set_size(outSize, rhoBatchSize); + hiddenLayer.set_size(outSize, rhoBatchSize); + outputGate.set_size(outSize, rhoBatchSize); - inputGateActivation.set_size(outSize, batchSize, bpttSteps); - forgetGateActivation.set_size(outSize, batchSize, bpttSteps); - outputGateActivation.set_size(outSize, batchSize, bpttSteps); - hiddenLayerActivation.set_size(outSize, batchSize, bpttSteps); + inputGateActivation.set_size(outSize, rhoBatchSize); + forgetGateActivation.set_size(outSize, rhoBatchSize); + outputGateActivation.set_size(outSize, rhoBatchSize); + hiddenLayerActivation.set_size(outSize, rhoBatchSize); - cellActivation.set_size(outSize, batchSize, bpttSteps); - outParameter.set_size(outSize, batchSize, bpttSteps); + cellActivation.set_size(outSize, rhoBatchSize); + prevError.set_size(4 * outSize, batchSize); // Now reset recurrent values to 0. - cell.zeros(outSize, batchSize, bpttSteps); + cell.zeros(outSize, size * batchSize); + outParameter.zeros(outSize, (size + 1) * batchSize); } -template -void LSTMType::SetWeights( - typename MatType::elem_type* weightsPtr) +template +void LSTM::Reset() { // Set the weight parameter for the output gate. - MakeAlias(input2GateOutputWeight, weightsPtr, outSize, inSize); - size_t offset = input2GateOutputWeight.n_elem; - MakeAlias(input2GateOutputBias, weightsPtr + offset, outSize, 1); - offset += input2GateOutputBias.n_elem; + input2GateOutputWeight = OutputDataType(weights.memptr(), outSize, inSize, + false, false); + input2GateOutputBias = OutputDataType(weights.memptr() + + input2GateOutputWeight.n_elem, outSize, 1, false, false); + size_t offset = input2GateOutputWeight.n_elem + input2GateOutputBias.n_elem; // Set the weight parameter for the forget gate. - MakeAlias(input2GateForgetWeight, weightsPtr + offset, outSize, inSize); - offset += input2GateForgetWeight.n_elem; - MakeAlias(input2GateForgetBias, weightsPtr + offset, outSize, 1); - offset += input2GateForgetBias.n_elem; + input2GateForgetWeight = OutputDataType(weights.memptr() + offset, + outSize, inSize, false, false); + input2GateForgetBias = OutputDataType(weights.memptr() + + offset + input2GateForgetWeight.n_elem, outSize, 1, false, false); + offset += input2GateForgetWeight.n_elem + input2GateForgetBias.n_elem; // Set the weight parameter for the input gate. - MakeAlias(input2GateInputWeight, weightsPtr + offset, outSize, inSize); - offset += input2GateInputWeight.n_elem; - MakeAlias(input2GateInputBias, weightsPtr + offset, outSize, 1); - offset += input2GateInputBias.n_elem; + input2GateInputWeight = OutputDataType(weights.memptr() + + offset, outSize, inSize, false, false); + input2GateInputBias = OutputDataType(weights.memptr() + + offset + input2GateInputWeight.n_elem, outSize, 1, false, false); + offset += input2GateInputWeight.n_elem + input2GateInputBias.n_elem; // Set the weight parameter for the hidden gate. - MakeAlias(input2HiddenWeight, weightsPtr + offset, outSize, inSize); - offset += input2HiddenWeight.n_elem; - MakeAlias(input2HiddenBias, weightsPtr + offset, outSize, 1); - offset += input2HiddenBias.n_elem; + input2HiddenWeight = OutputDataType(weights.memptr() + + offset, outSize, inSize, false, false); + input2HiddenBias = OutputDataType(weights.memptr() + + offset + input2HiddenWeight.n_elem, outSize, 1, false, false); + offset += input2HiddenWeight.n_elem + input2HiddenBias.n_elem; // Set the weight parameter for the output multiplication. - MakeAlias(output2GateOutputWeight, weightsPtr + offset, outSize, outSize); + output2GateOutputWeight = OutputDataType(weights.memptr() + + offset, outSize, outSize, false, false); offset += output2GateOutputWeight.n_elem; // Set the weight parameter for the output multiplication. - MakeAlias(output2GateForgetWeight, weightsPtr + offset, outSize, outSize); + output2GateForgetWeight = OutputDataType(weights.memptr() + + offset, outSize, outSize, false, false); offset += output2GateForgetWeight.n_elem; // Set the weight parameter for the input multiplication. - MakeAlias(output2GateInputWeight, weightsPtr + offset, outSize, outSize); + output2GateInputWeight = OutputDataType(weights.memptr() + + offset, outSize, outSize, false, false); offset += output2GateInputWeight.n_elem; // Set the weight parameter for the hidden multiplication. - MakeAlias(output2HiddenWeight, weightsPtr + offset, outSize, outSize); + output2HiddenWeight = OutputDataType(weights.memptr() + + offset, outSize, outSize, false, false); offset += output2HiddenWeight.n_elem; // Set the weight parameter for the cell multiplication. - MakeAlias(cell2GateOutputWeight, weightsPtr + offset, outSize, 1); + cell2GateOutputWeight = OutputDataType(weights.memptr() + + offset, outSize, 1, false, false); offset += cell2GateOutputWeight.n_elem; // Set the weight parameter for the cell - forget gate multiplication. - MakeAlias(cell2GateForgetWeight, weightsPtr + offset, outSize, 1); + cell2GateForgetWeight = OutputDataType(weights.memptr() + + offset, outSize, 1, false, false); offset += cell2GateOutputWeight.n_elem; // Set the weight parameter for the cell - input gate multiplication. - MakeAlias(cell2GateInputWeight, weightsPtr + offset, outSize, 1); + cell2GateInputWeight = OutputDataType(weights.memptr() + + offset, outSize, 1, false, false); } // Forward when cellState is not needed. -template -void LSTMType::Forward(const MatType& input, MatType& output) +template +template +void LSTM::Forward( + const InputType& input, OutputType& output) { - // Convenience alias. - const size_t batchSize = input.n_cols; + //! Locally-stored cellState. + OutputType cellState; + Forward(input, output, cellState, false); +} - inputGate = input2GateInputWeight * input; - if (this->HasPreviousStep()) +// Forward when cellState is needed overloaded LSTM::Forward(). +template +template +void LSTM::Forward(const InputType& input, + OutputType& output, + OutputType& cellState, + bool useCellState) +{ + // Check if the batch size changed, the number of cols is defines the input + // batch size. + if (input.n_cols != batchSize) { - inputGate += - output2GateInputWeight * outParameter.slice(this->PreviousStep()); - } - inputGate.each_col() += input2GateInputBias; - - forgetGate = input2GateForgetWeight * input; - if (this->HasPreviousStep()) - { - forgetGate += output2GateForgetWeight * outParameter.slice( - this->PreviousStep()); - } - forgetGate.each_col() += input2GateForgetBias; - - if (this->HasPreviousStep()) - { - inputGate += arma::repmat(cell2GateInputWeight, 1, batchSize) % - cell.slice(this->PreviousStep()); - - forgetGate += arma::repmat(cell2GateForgetWeight, 1, batchSize) % - cell.slice(this->PreviousStep()); + batchSize = input.n_cols; + batchStep = batchSize - 1; + ResetCell(rhoSize); } - inputGateActivation.slice(this->CurrentStep()) = - 1.0 / (1.0 + arma::exp(-inputGate)); - forgetGateActivation.slice(this->CurrentStep()) = - 1.0 / (1.0 + arma::exp(-forgetGate)); + inputGate.cols(forwardStep, forwardStep + batchStep) = input2GateInputWeight * + input + output2GateInputWeight * outParameter.cols(forwardStep, + forwardStep + batchStep); + inputGate.cols(forwardStep, forwardStep + batchStep).each_col() += + input2GateInputBias; - hiddenLayer = input2HiddenWeight * input; - if (this->HasPreviousStep()) + forgetGate.cols(forwardStep, forwardStep + batchStep) = input2GateForgetWeight + * input + output2GateForgetWeight * outParameter.cols( + forwardStep, forwardStep + batchStep); + forgetGate.cols(forwardStep, forwardStep + batchStep).each_col() += + input2GateForgetBias; + + if (forwardStep > 0) { - hiddenLayer += output2HiddenWeight * - outParameter.slice(this->PreviousStep()); + if (useCellState) + { + if (!cellState.is_empty()) + { + cell.cols(forwardStep - batchSize, + forwardStep - batchSize + batchStep) = cellState; + } + else + { + throw std::runtime_error("Cell parameter is empty."); + } + } + inputGate.cols(forwardStep, forwardStep + batchStep) += + arma::repmat(cell2GateInputWeight, 1, batchSize) % + cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep); + + forgetGate.cols(forwardStep, forwardStep + batchStep) += + arma::repmat(cell2GateForgetWeight, 1, batchSize) % + cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep); } - hiddenLayer.each_col() += input2HiddenBias; - hiddenLayerActivation.slice(this->CurrentStep()) = arma::tanh(hiddenLayer); + inputGateActivation.cols(forwardStep, forwardStep + batchStep) = 1.0 / + (1 + arma::exp(-inputGate.cols(forwardStep, forwardStep + batchStep))); - if (!this->HasPreviousStep()) + forgetGateActivation.cols(forwardStep, forwardStep + batchStep) = 1.0 / + (1 + arma::exp(-forgetGate.cols(forwardStep, forwardStep + batchStep))); + + hiddenLayer.cols(forwardStep, forwardStep + batchStep) = input2HiddenWeight * + input + output2HiddenWeight * outParameter.cols( + forwardStep, forwardStep + batchStep); + + hiddenLayer.cols(forwardStep, forwardStep + batchStep).each_col() += + input2HiddenBias; + + hiddenLayerActivation.cols(forwardStep, forwardStep + batchStep) = + arma::tanh(hiddenLayer.cols(forwardStep, forwardStep + batchStep)); + + if (forwardStep == 0) { - cell.slice(this->CurrentStep()) = - inputGateActivation.slice(this->CurrentStep()) % - hiddenLayerActivation.slice(this->CurrentStep()); + cell.cols(forwardStep, forwardStep + batchStep) = + inputGateActivation.cols(forwardStep, forwardStep + batchStep) % + hiddenLayerActivation.cols(forwardStep, forwardStep + batchStep); } else { - cell.slice(this->CurrentStep()) = - forgetGateActivation.slice(this->CurrentStep()) % - cell.slice(this->PreviousStep()) + - inputGateActivation.slice(this->CurrentStep()) % - hiddenLayerActivation.slice(this->CurrentStep()); + cell.cols(forwardStep, forwardStep + batchStep) = + forgetGateActivation.cols(forwardStep, forwardStep + batchStep) % + cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep) + + inputGateActivation.cols(forwardStep, forwardStep + batchStep) % + hiddenLayerActivation.cols(forwardStep, forwardStep + batchStep); } - outputGate = input2GateOutputWeight * input + - cell.slice(this->CurrentStep()).each_col() % cell2GateOutputWeight; - if (this->HasPreviousStep()) + outputGate.cols(forwardStep, forwardStep + batchStep) = input2GateOutputWeight + * input + output2GateOutputWeight * outParameter.cols( + forwardStep, forwardStep + batchStep) + cell.cols(forwardStep, + forwardStep + batchStep).each_col() % cell2GateOutputWeight; + + outputGate.cols(forwardStep, forwardStep + batchStep).each_col() += + input2GateOutputBias; + + outputGateActivation.cols(forwardStep, forwardStep + batchStep) = 1.0 / + (1 + arma::exp(-outputGate.cols(forwardStep, forwardStep + batchStep))); + + cellActivation.cols(forwardStep, forwardStep + batchStep) = + arma::tanh(cell.cols(forwardStep, forwardStep + batchStep)); + + outParameter.cols(forwardStep + batchSize, + forwardStep + batchSize + batchStep) = + cellActivation.cols(forwardStep, forwardStep + batchStep) % + outputGateActivation.cols(forwardStep, forwardStep + batchStep); + + output = OutputType(outParameter.memptr() + + (forwardStep + batchSize) * outSize, outSize, batchSize, false, false); + + cellState = OutputType(cell.memptr() + + forwardStep * outSize, outSize, batchSize, false, false); + + forwardStep += batchSize; + if ((forwardStep / batchSize) == bpttSteps) { - outputGate += - output2GateOutputWeight * outParameter.slice(this->PreviousStep()); + forwardStep = 0; } - outputGate.each_col() += input2GateOutputBias; - - outputGateActivation.slice(this->CurrentStep()) = - 1.0 / (1.0 + arma::exp(-outputGate)); - - cellActivation.slice(this->CurrentStep()) = - arma::tanh(cell.slice(this->CurrentStep())); - - // There's a bit of an issue here: we need to preserve the output for the next - // time step, but we also need to set `output` to that. Unfortunately for now - // we make a copy, but it's possible that we could instead use an alias here, - // or have `outParameter` hold a collection of aliases. - outParameter.slice(this->CurrentStep()) = - cellActivation.slice(this->CurrentStep()) % - outputGateActivation.slice(this->CurrentStep()); - - output = outParameter.slice(this->CurrentStep()); } -template -void LSTMType::Backward( - const MatType& /* input */, const MatType& gy, MatType& g) +template +template +void LSTM::Backward( + const InputType& /* input */, const ErrorType& gy, GradientType& g) { - MatType gyLocal; - if (this->HasPreviousStep()) + ErrorType gyLocal; + if (gradientStepIdx > 0) { - gyLocal = gy + output2GateOutputWeight.t() * outputGateError + - output2GateForgetWeight.t() * forgetGateError + - output2GateInputWeight.t() * inputGateError + - output2HiddenWeight.t() * hiddenError; + gyLocal = gy + prevError; } else { // Make an alias. - gyLocal = MatType(((MatType&) gy).memptr(), gy.n_rows, gy.n_cols, - false, false); + gyLocal = ErrorType(((ErrorType&) gy).memptr(), gy.n_rows, gy.n_cols, false, + false); } - outputGateError = gyLocal % cellActivation.slice(this->CurrentStep()) % - (outputGateActivation.slice(this->CurrentStep()) % - (1.0 - outputGateActivation.slice(this->CurrentStep()))); + outputGateError = + gyLocal % cellActivation.cols(backwardStep - batchStep, backwardStep) % + (outputGateActivation.cols(backwardStep - batchStep, backwardStep) % + (1.0 - outputGateActivation.cols(backwardStep - batchStep, + backwardStep))); - MatType cellError = gyLocal % - outputGateActivation.slice(this->CurrentStep()) % - (1 - arma::pow(cellActivation.slice(this->CurrentStep()), 2)) + - outputGateError.each_col() % cell2GateOutputWeight; + OutputDataType cellError = gyLocal % + outputGateActivation.cols(backwardStep - batchStep, backwardStep) % + (1 - arma::pow(cellActivation.cols(backwardStep - + batchStep, backwardStep), 2)) + outputGateError.each_col() % + cell2GateOutputWeight; - if (this->HasPreviousStep()) + if (gradientStepIdx > 0) { cellError += inputCellError; } - if (this->HasPreviousStep()) + if (backwardStep > batchStep) { - forgetGateError = cell.slice(this->PreviousStep()) % cellError % - (forgetGateActivation.slice(this->CurrentStep()) % - (1.0 - forgetGateActivation.slice(this->CurrentStep()))); + forgetGateError = cell.cols((backwardStep - batchSize) - batchStep, + (backwardStep - batchSize)) % cellError % (forgetGateActivation.cols( + backwardStep - batchStep, backwardStep) % (1.0 - + forgetGateActivation.cols(backwardStep - batchStep, backwardStep))); } else { - forgetGateError.zeros(forgetGateActivation.n_rows, - forgetGateActivation.n_cols); + forgetGateError.zeros(); } - inputGateError = hiddenLayerActivation.slice(this->CurrentStep()) % - cellError % (inputGateActivation.slice(this->CurrentStep()) % - (1.0 - inputGateActivation.slice(this->CurrentStep()))); + inputGateError = hiddenLayerActivation.cols(backwardStep - batchStep, + backwardStep) % cellError % + (inputGateActivation.cols(backwardStep - batchStep, backwardStep) % + (1.0 - inputGateActivation.cols(backwardStep - batchStep, backwardStep))); - hiddenError = inputGateActivation.slice(this->CurrentStep()) % cellError % - (1 - arma::pow(hiddenLayerActivation.slice(this->CurrentStep()), 2)); + hiddenError = inputGateActivation.cols(backwardStep - batchStep, + backwardStep) % cellError % (1 - arma::pow(hiddenLayerActivation.cols( + backwardStep - batchStep, backwardStep), 2)); - inputCellError = forgetGateActivation.slice(this->CurrentStep()) % cellError + - forgetGateError.each_col() % cell2GateForgetWeight + - inputGateError.each_col() % cell2GateInputWeight; + inputCellError = forgetGateActivation.cols(backwardStep - batchStep, + backwardStep) % cellError + forgetGateError.each_col() % + cell2GateForgetWeight + inputGateError.each_col() % cell2GateInputWeight; g = input2GateInputWeight.t() * inputGateError + input2HiddenWeight.t() * hiddenError + input2GateForgetWeight.t() * forgetGateError + input2GateOutputWeight.t() * outputGateError; + + prevError = output2GateOutputWeight.t() * outputGateError + + output2GateForgetWeight.t() * forgetGateError + + output2GateInputWeight.t() * inputGateError + + output2HiddenWeight.t() * hiddenError; + + backwardStep -= batchSize; + gradientStepIdx++; + if (gradientStepIdx == bpttSteps) + { + backwardStep = bpttSteps - 1; + gradientStepIdx = 0; + } } -template -void LSTMType::Gradient( - const MatType& input, - const MatType& /* error */, - MatType& gradient) +template +template +void LSTM::Gradient( + const InputType& input, + const ErrorType& /* error */, + GradientType& gradient) { - // This implementation depends on Gradient() being called just after - // Backward(), which is something we can safely assume. - // Input2GateOutputWeight and input2GateOutputBias gradients. gradient.submat(0, 0, input2GateOutputWeight.n_elem - 1, 0) = arma::vectorise(outputGateError * input.t()); @@ -316,7 +450,7 @@ void LSTMType::Gradient( arma::sum(outputGateError, 1); size_t offset = input2GateOutputWeight.n_elem + input2GateOutputBias.n_elem; - // input2GateForgetWeight and input2GateForgetBias gradients. + // Input2GateForgetWeight and input2GateForgetBias gradients. gradient.submat(offset, 0, offset + input2GateForgetWeight.n_elem - 1, 0) = arma::vectorise(forgetGateError * input.t()); gradient.submat(offset + input2GateForgetWeight.n_elem, 0, @@ -324,7 +458,7 @@ void LSTMType::Gradient( input2GateForgetBias.n_elem - 1, 0) = arma::sum(forgetGateError, 1); offset += input2GateForgetWeight.n_elem + input2GateForgetBias.n_elem; - // input2GateInputWeight and input2GateInputBias gradients. + // Input2GateInputWeight and input2GateInputBias gradients. gradient.submat(offset, 0, offset + input2GateInputWeight.n_elem - 1, 0) = arma::vectorise(inputGateError * input.t()); gradient.submat(offset + input2GateInputWeight.n_elem, 0, @@ -332,7 +466,7 @@ void LSTMType::Gradient( input2GateInputBias.n_elem - 1, 0) = arma::sum(inputGateError, 1); offset += input2GateInputWeight.n_elem + input2GateInputBias.n_elem; - // input2HiddenWeight and input2HiddenBias gradients. + // Input2HiddenWeight and input2HiddenBias gradients. gradient.submat(offset, 0, offset + input2HiddenWeight.n_elem - 1, 0) = arma::vectorise(hiddenError * input.t()); gradient.submat(offset + input2HiddenWeight.n_elem, 0, @@ -340,43 +474,48 @@ void LSTMType::Gradient( arma::sum(hiddenError, 1); offset += input2HiddenWeight.n_elem + input2HiddenBias.n_elem; - // output2GateOutputWeight gradients. + // Output2GateOutputWeight gradients. gradient.submat(offset, 0, offset + output2GateOutputWeight.n_elem - 1, 0) = arma::vectorise(outputGateError * - outParameter.slice(this->CurrentStep()).t()); + outParameter.cols(gradientStep - batchStep, gradientStep).t()); offset += output2GateOutputWeight.n_elem; - // output2GateForgetWeight gradients. + // Output2GateForgetWeight gradients. gradient.submat(offset, 0, offset + output2GateForgetWeight.n_elem - 1, 0) = arma::vectorise(forgetGateError * - outParameter.slice(this->CurrentStep()).t()); + outParameter.cols(gradientStep - batchStep, gradientStep).t()); offset += output2GateForgetWeight.n_elem; - // output2GateInputWeight gradients. + // Output2GateInputWeight gradients. gradient.submat(offset, 0, offset + output2GateInputWeight.n_elem - 1, 0) = arma::vectorise(inputGateError * - outParameter.slice(this->CurrentStep()).t()); + outParameter.cols(gradientStep - batchStep, gradientStep).t()); offset += output2GateInputWeight.n_elem; - // output2HiddenWeight gradients. + // Output2HiddenWeight gradients. gradient.submat(offset, 0, offset + output2HiddenWeight.n_elem - 1, 0) = arma::vectorise(hiddenError * - outParameter.slice(this->CurrentStep()).t()); + outParameter.cols(gradientStep - batchStep, gradientStep).t()); offset += output2HiddenWeight.n_elem; - // cell2GateOutputWeight gradients. + // Cell2GateOutputWeight gradients. gradient.submat(offset, 0, offset + cell2GateOutputWeight.n_elem - 1, 0) = - arma::sum(outputGateError % cell.slice(this->CurrentStep()), 1); + arma::sum(outputGateError % + cell.cols(gradientStep - batchStep, gradientStep), 1); offset += cell2GateOutputWeight.n_elem; - // cell2GateForgetWeight and cell2GateInputWeight gradients. - if (this->HasPreviousStep()) + // Cell2GateForgetWeight and cell2GateInputWeight gradients. + if (gradientStep > batchStep) { gradient.submat(offset, 0, offset + cell2GateForgetWeight.n_elem - 1, 0) = - arma::sum(forgetGateError % cell.slice(this->PreviousStep()), 1); + arma::sum(forgetGateError % + cell.cols((gradientStep - batchSize) - batchStep, + (gradientStep - batchSize)), 1); gradient.submat(offset + cell2GateForgetWeight.n_elem, 0, offset + cell2GateForgetWeight.n_elem + cell2GateInputWeight.n_elem - 1, 0) = - arma::sum(inputGateError % cell.slice(this->PreviousStep()), 1); + arma::sum(inputGateError % + cell.cols((gradientStep - batchSize) - batchStep, + (gradientStep - batchSize)), 1); } else { @@ -386,32 +525,41 @@ void LSTMType::Gradient( cell2GateForgetWeight.n_elem + cell2GateInputWeight.n_elem - 1, 0).zeros(); } + + if (gradientStep == 0) + { + gradientStep = batchSize * bpttSteps - 1; + } + else + { + gradientStep -= batchSize; + } } -template +template template -void LSTMType::serialize(Archive& ar, const uint32_t /* version */) +void LSTM::serialize( + Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - + ar(CEREAL_NVP(weights)); ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(outSize)); - - // Clear recurrent state if we are loading. - if (Archive::is_loading::value) - { - inputGateActivation.clear(); - forgetGateActivation.clear(); - outputGateActivation.clear(); - hiddenLayerActivation.clear(); - cellActivation.clear(); - forgetGateError.clear(); - outputGateError.clear(); - outParameter.clear(); - inputCellError.clear(); - inputGateError.clear(); - hiddenError.clear(); - } + ar(CEREAL_NVP(rho)); + ar(CEREAL_NVP(bpttSteps)); + ar(CEREAL_NVP(batchSize)); + ar(CEREAL_NVP(batchStep)); + ar(CEREAL_NVP(forwardStep)); + ar(CEREAL_NVP(backwardStep)); + ar(CEREAL_NVP(gradientStep)); + ar(CEREAL_NVP(gradientStepIdx)); + ar(CEREAL_NVP(cell)); + ar(CEREAL_NVP(inputGateActivation)); + ar(CEREAL_NVP(forgetGateActivation)); + ar(CEREAL_NVP(outputGateActivation)); + ar(CEREAL_NVP(hiddenLayerActivation)); + ar(CEREAL_NVP(cellActivation)); + ar(CEREAL_NVP(prevError)); + ar(CEREAL_NVP(outParameter)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/max_pooling.hpp b/src/mlpack/methods/ann/layer/max_pooling.hpp index 25d25509e4..2547c5596a 100644 --- a/src/mlpack/methods/ann/layer/max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling.hpp @@ -15,8 +15,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -30,39 +28,32 @@ class MaxPoolingRule /* * Return the maximum value within the receptive block. * - * @param input Input used to perform the pooling operation. Could be an - * Armadillo subview. + * @param input Input used to perform the pooling operation. */ template - typename MatType::elem_type Pooling(const MatType& input) + size_t Pooling(const MatType& input) { - return arma::max(arma::vectorise(input)); - } - - template - std::tuple PoolingWithIndex( - const MatType& input) - { - const typename MatType::elem_type maxVal = - arma::max(arma::vectorise(input)); - const size_t index = arma::as_scalar(arma::find(input == maxVal, 1)); - - return std::tuple(index, maxVal); + return arma::as_scalar(arma::find(input.max() == input, 1)); } }; /** * Implementation of the MaxPooling layer. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class MaxPoolingType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MaxPooling { public: //! Create the MaxPooling object. - MaxPoolingType(); + MaxPooling(); /** * Create the MaxPooling object using the specified number of units. @@ -73,25 +64,11 @@ class MaxPoolingType : public Layer * @param strideHeight Width of the stride operation. * @param floor Rounding operator (floor or ceil). */ - MaxPoolingType(const size_t kernelWidth, - const size_t kernelHeight, - const size_t strideWidth = 1, - const size_t strideHeight = 1, - const bool floor = true); - - // Virtual destructor. - virtual ~MaxPoolingType() { } - - //! Copy the given MaxPoolingType. - MaxPoolingType(const MaxPoolingType& other); - //! Take ownership of the given MaxPoolingType. - MaxPoolingType(MaxPoolingType&& other); - //! Copy the given MaxPoolingType. - MaxPoolingType& operator=(const MaxPoolingType& other); - //! Take ownership of the given MaxPoolingType. - MaxPoolingType& operator=(MaxPoolingType&& other); - - MaxPoolingType* Clone() const { return new MaxPoolingType(*this); } + MaxPooling(const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth = 1, + const size_t strideHeight = 1, + const bool floor = true); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -100,7 +77,8 @@ class MaxPoolingType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, using 3rd-order tensors as @@ -111,37 +89,79 @@ class MaxPoolingType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, - const MatType& gy, - MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + const OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + const OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the input width. + size_t InputWidth() const { return inputWidth; } + //! Modify the input width. + size_t& InputWidth() { return inputWidth; } + + //! Get the input height. + size_t InputHeight() const { return inputHeight; } + //! Modify the input height. + size_t& InputHeight() { return inputHeight; } + + //! Get the output width. + size_t OutputWidth() const { return outputWidth; } + //! Modify the output width. + size_t& OutputWidth() { return outputWidth; } + + //! Get the output height. + size_t OutputHeight() const { return outputHeight; } + //! Modify the output height. + size_t& OutputHeight() { return outputHeight; } + + //! Get the input size. + size_t InputSize() const { return inSize; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } //! Get the kernel width. - size_t const& KernelWidth() const { return kernelWidth; } + size_t KernelWidth() const { return kernelWidth; } //! Modify the kernel width. size_t& KernelWidth() { return kernelWidth; } //! Get the kernel height. - size_t const& KernelHeight() const { return kernelHeight; } + size_t KernelHeight() const { return kernelHeight; } //! Modify the kernel height. size_t& KernelHeight() { return kernelHeight; } //! Get the stride width. - size_t const& StrideWidth() const { return strideWidth; } + size_t StrideWidth() const { return strideWidth; } //! Modify the stride width. size_t& StrideWidth() { return strideWidth; } //! Get the stride height. - size_t const& StrideHeight() const { return strideHeight; } + size_t StrideHeight() const { return strideHeight; } //! Modify the stride height. size_t& StrideHeight() { return strideHeight; } //! Get the value of the rounding operation. - bool const& Floor() const { return floor; } + bool Floor() const { return floor; } //! Modify the value of the rounding operation. bool& Floor() { return floor; } - //! Compute the size of the output given `InputDimensions()`. - void ComputeOutputDimensions(); + //! Get the value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } + + //! Get the size of the weights. + size_t WeightSize() const { return 0; } /** * Serialize the layer. @@ -150,94 +170,62 @@ class MaxPoolingType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: - /** + /** * Apply pooling to the input and store the results. * * @param input The input to be apply the pooling rule. * @param output The pooled result. * @param poolingIndices The pooled indices. */ - void PoolingOperation( - const arma::Cube& input, - arma::Cube& output, - arma::Cube& poolingIndices) + template + void PoolingOperation(const arma::Mat& input, + arma::Mat& output, + arma::Mat& poolingIndices) { - // Iterate over all slices individually. - for (size_t s = 0; s < input.n_slices; ++s) + for (size_t j = 0, colidx = 0; j < output.n_cols; + ++j, colidx += strideHeight) { - for (size_t j = 0, colidx = 0; j < output.n_cols; - ++j, colidx += strideHeight) + for (size_t i = 0, rowidx = 0; i < output.n_rows; + ++i, rowidx += strideWidth) { - for (size_t i = 0, rowidx = 0; i < output.n_rows; - ++i, rowidx += strideWidth) + size_t rowEnd = rowidx + kernelWidth - 1; + size_t colEnd = colidx + kernelHeight - 1; + + if (rowEnd > input.n_rows - 1) + rowEnd = input.n_rows - 1; + if (colEnd > input.n_cols - 1) + colEnd = input.n_cols - 1; + + arma::mat subInput = input( + arma::span(rowidx, rowEnd), + arma::span(colidx, colEnd)); + + const size_t idx = pooling.Pooling(subInput); + output(i, j) = subInput(idx); + + if (!deterministic) { - const std::tuple poolResult = - pooling.PoolingWithIndex(input.slice(s).submat( - rowidx, - colidx, - rowidx + kernelWidth - 1 - offset, - colidx + kernelHeight - 1 - offset)); + arma::Mat subIndices = indices(arma::span(rowidx, rowEnd), + arma::span(colidx, colEnd)); - // Now map the returned pooling index, which corresponds to the - // submatrix we gave, back to its position in the (linearized) input. - const size_t poolIndex = std::get<0>(poolResult); - const size_t poolingCol = poolIndex / (kernelWidth - offset); - const size_t poolingRow = poolIndex % (kernelWidth - offset); - const size_t unmappedPoolingIndex = (rowidx + poolingRow) + - input.n_rows * (colidx + poolingCol) + - input.n_rows * input.n_cols * s; - - poolingIndices(i, j, s) = unmappedPoolingIndex; - output(i, j, s) = std::get<1>(poolResult); + poolingIndices(i, j) = subIndices(idx); } } } } /** - * Apply pooling to all slices of the input and store the results, but not the - * indices used. - * - * @param input The input to apply the pooling rule to. - * @param output The pooled result. - */ - void PoolingOperation( - const arma::Cube& input, - arma::Cube& output) - { - // Iterate over all slices individually. - for (size_t s = 0; s < input.n_slices; ++s) - { - for (size_t j = 0, colidx = 0; j < output.n_cols; - ++j, colidx += strideHeight) - { - for (size_t i = 0, rowidx = 0; i < output.n_rows; - ++i, rowidx += strideWidth) - { - output(i, j, s) = pooling.Pooling(input.slice(s).submat( - rowidx, - colidx, - rowidx + kernelWidth - 1 - offset, - colidx + kernelHeight - 1 - offset)); - } - } - } - } - - /** - * Apply unpooling to all slices of the input and store the results. + * Apply unpooling to the input and store the results. * * @param error The backward error. * @param output The pooled result. - * @param poolingIndices The pooled indices (from `PoolingOperation()`). + * @param poolingIndices The pooled indices. */ - void UnpoolingOperation( - const arma::Cube& error, - arma::Cube& output, - const arma::Cube& poolingIndices) + template + void Unpooling(const arma::Mat& error, + arma::Mat& output, + arma::Mat& poolingIndices) { - output.zeros(); - for (size_t i = 0; i < poolingIndices.n_elem; ++i) { output(poolingIndices(i)) += error(i); @@ -259,22 +247,64 @@ class MaxPoolingType : public Layer //! Rounding operation used. bool floor; - //! Locally-stored number of channels. - size_t channels; + //! Locally-stored number of input channels. + size_t inSize; - //! Locally-stored offset: indicates whether we take the first element or the - //! second element when pooling. Computed by `ComputeOutputDimensions()`. - size_t offset; + //! Locally-stored number of output channels. + size_t outSize; + + //! Locally-stored reset parameter used to initialize the module once. + bool reset; + + //! Locally-stored input width. + size_t inputWidth; + + //! Locally-stored input height. + size_t inputHeight; + + //! Locally-stored output width. + size_t outputWidth; + + //! Locally-stored output height. + size_t outputHeight; + + //! If true use maximum a posteriori during the forward pass. + bool deterministic; + + + //! Locally-stored number of input units. + size_t batchSize; + + //! Locally-stored output parameter. + arma::cube outputTemp; + + //! Locally-stored transformed input parameter. + arma::cube inputTemp; + + //! Locally-stored transformed output parameter. + arma::cube gTemp; //! Locally-stored pooling strategy. MaxPoolingRule pooling; - //! Locally-stored pooling indices. - arma::Cube poolingIndices; -}; // class MaxPoolingType + //! Locally-stored delta object. + OutputDataType delta; -// Standard MaxPooling layer. -typedef MaxPoolingType MaxPooling; + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored indices matrix parameter. + arma::Mat indices; + + //! Locally-stored indices column parameter. + arma::Col indicesCol; + + //! Locally-stored pooling indicies. + std::vector poolingIndices; +}; // class MaxPooling } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp index 29badb4e36..9650a5f2c3 100644 --- a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp @@ -19,201 +19,141 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -MaxPoolingType::MaxPoolingType() : - Layer() +template +MaxPooling::MaxPooling() { // Nothing to do here. } -template -MaxPoolingType::MaxPoolingType( +template +MaxPooling::MaxPooling( const size_t kernelWidth, const size_t kernelHeight, const size_t strideWidth, const size_t strideHeight, const bool floor) : - Layer(), kernelWidth(kernelWidth), kernelHeight(kernelHeight), strideWidth(strideWidth), strideHeight(strideHeight), floor(floor), - channels(0), - offset(0) + inSize(0), + outSize(0), + reset(false), + inputWidth(0), + inputHeight(0), + outputWidth(0), + outputHeight(0), + deterministic(false), + batchSize(0) { // Nothing to do here. } -template -MaxPoolingType::MaxPoolingType( - const MaxPoolingType& other) : - Layer(other), - kernelWidth(other.kernelWidth), - kernelHeight(other.kernelHeight), - strideWidth(other.strideWidth), - strideHeight(other.strideHeight), - floor(other.floor), - channels(other.channels), - offset(other.offset), - pooling(other.pooling) +template +template +void MaxPooling::Forward( + const arma::Mat& input, arma::Mat& output) { - // Nothing to do here. -} + batchSize = input.n_cols; + inSize = input.n_elem / (inputWidth * inputHeight * batchSize); + inputTemp = arma::cube(const_cast&>(input).memptr(), + inputWidth, inputHeight, batchSize * inSize, false, false); -template -MaxPoolingType::MaxPoolingType( - MaxPoolingType&& other) : - Layer(std::move(other)), - kernelWidth(std::move(other.kernelWidth)), - kernelHeight(std::move(other.kernelHeight)), - strideWidth(std::move(other.strideWidth)), - strideHeight(std::move(other.strideHeight)), - floor(std::move(other.floor)), - channels(std::move(other.channels)), - offset(std::move(other.offset)), - pooling(std::move(other.pooling)) -{ - // Nothing to do here. -} - -template -MaxPoolingType& -MaxPoolingType::operator=(const MaxPoolingType& other) -{ - if (&other != this) - { - Layer::operator=(other); - kernelWidth = other.kernelWidth; - kernelHeight = other.kernelHeight; - strideWidth = other.strideWidth; - strideHeight = other.strideHeight; - floor = other.floor; - channels = other.channels; - offset = other.offset; - pooling = other.pooling; - } - - return *this; -} - -template -MaxPoolingType& -MaxPoolingType::operator=(MaxPoolingType&& other) -{ - if (&other != this) - { - Layer::operator=(std::move(other)); - kernelWidth = std::move(other.kernelWidth); - kernelHeight = std::move(other.kernelHeight); - strideWidth = std::move(other.strideWidth); - strideHeight = std::move(other.strideHeight); - floor = std::move(other.floor); - channels = std::move(other.channels); - offset = std::move(other.offset); - pooling = std::move(other.pooling); - } - - return *this; -} - -template -void MaxPoolingType::Forward(const MatType& input, MatType& output) -{ - arma::Cube inputTemp( - const_cast(input).memptr(), this->inputDimensions[0], - this->inputDimensions[1], input.n_cols * channels, false, false); - - arma::Cube outputTemp(output.memptr(), - this->outputDimensions[0], this->outputDimensions[1], - input.n_cols * channels, false, true); - - if (this->training) - { - // If we are training, we'll do a backwards pass, so we need to ensure that - // we know what indices we used. - poolingIndices.set_size(this->outputDimensions[0], - this->outputDimensions[1], input.n_cols * channels); - - PoolingOperation(inputTemp, outputTemp, poolingIndices); - } - else - { - PoolingOperation(inputTemp, outputTemp); - } -} - -template -void MaxPoolingType::Backward( - const MatType& input, const MatType& gy, MatType& g) -{ - arma::Cube mappedError = - arma::Cube(((MatType&) gy).memptr(), - this->outputDimensions[0], this->outputDimensions[1], - channels * input.n_cols, false, false); - - arma::Cube gTemp(g.memptr(), - this->inputDimensions[0], this->inputDimensions[1], - channels * input.n_cols, false, true); - - // There's no version of UnpoolingOperation without pooling indices, because - // if we call `Backward()`, we know for sure we are training. - UnpoolingOperation(mappedError, gTemp, poolingIndices); -} - -template -void MaxPoolingType::ComputeOutputDimensions() -{ - this->outputDimensions = this->inputDimensions; - - // Compute the size of the output. if (floor) { - this->outputDimensions[0] = std::floor((this->inputDimensions[0] - + outputWidth = std::floor((inputWidth - (double) kernelWidth) / (double) strideWidth + 1); - this->outputDimensions[1] = std::floor((this->inputDimensions[1] - + outputHeight = std::floor((inputHeight - (double) kernelHeight) / (double) strideHeight + 1); - offset = 0; } else { - this->outputDimensions[0] = std::ceil((this->inputDimensions[0] - + outputWidth = std::ceil((inputWidth - (double) kernelWidth) / (double) strideWidth + 1); - this->outputDimensions[1] = std::ceil((this->inputDimensions[1] - + outputHeight = std::ceil((inputHeight - (double) kernelHeight) / (double) strideHeight + 1); - offset = 1; } - // Higher dimensions are not modified. + outputTemp = arma::zeros >(outputWidth, outputHeight, + batchSize * inSize); - // Cache input size and output size. - channels = 1; - for (size_t i = 2; i < this->inputDimensions.size(); ++i) - channels *= this->inputDimensions[i]; + if (!deterministic) + { + poolingIndices.push_back(outputTemp); + } + + if (!reset) + { + size_t elements = inputWidth * inputHeight; + indicesCol = arma::linspace >(0, (elements - 1), + elements); + + indices = arma::Mat(indicesCol.memptr(), inputWidth, inputHeight); + + reset = true; + } + + for (size_t s = 0; s < inputTemp.n_slices; s++) + { + if (!deterministic) + { + PoolingOperation(inputTemp.slice(s), outputTemp.slice(s), + poolingIndices.back().slice(s)); + } + else + { + PoolingOperation(inputTemp.slice(s), outputTemp.slice(s), + inputTemp.slice(s)); + } + } + + output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem / batchSize, + batchSize); + + outputWidth = outputTemp.n_rows; + outputHeight = outputTemp.n_cols; + outSize = batchSize * inSize; } -template +template +template +void MaxPooling::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) +{ + arma::cube mappedError = arma::cube(((arma::Mat&) gy).memptr(), + outputWidth, outputHeight, outSize, false, false); + + gTemp = arma::zeros(inputTemp.n_rows, + inputTemp.n_cols, inputTemp.n_slices); + + for (size_t s = 0; s < mappedError.n_slices; s++) + { + Unpooling(mappedError.slice(s), gTemp.slice(s), + poolingIndices.back().slice(s)); + } + + poolingIndices.pop_back(); + + g = arma::mat(gTemp.memptr(), gTemp.n_elem / batchSize, batchSize); +} + +template template -void MaxPoolingType::serialize( +void MaxPooling::serialize( Archive& ar, const uint32_t /* version */) - { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(kernelWidth)); ar(CEREAL_NVP(kernelHeight)); ar(CEREAL_NVP(strideWidth)); ar(CEREAL_NVP(strideHeight)); - ar(CEREAL_NVP(channels)); + ar(CEREAL_NVP(batchSize)); ar(CEREAL_NVP(floor)); - ar(CEREAL_NVP(offset)); - - if (Archive::is_loading::value) - { - // Clear any memory used by `poolingIndices`. - poolingIndices.clear(); - } + ar(CEREAL_NVP(inputWidth)); + ar(CEREAL_NVP(inputHeight)); + ar(CEREAL_NVP(outputWidth)); + ar(CEREAL_NVP(outputHeight)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/mean_pooling.hpp b/src/mlpack/methods/ann/layer/mean_pooling.hpp similarity index 73% rename from src/mlpack/methods/ann/layer/not_adapted/mean_pooling.hpp rename to src/mlpack/methods/ann/layer/mean_pooling.hpp index 7973fc1aa5..4156667beb 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/mean_pooling.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling.hpp @@ -21,20 +21,20 @@ namespace ann /** Artificial Neural Network. */ { /** * Implementation of the MeanPooling. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class MeanPoolingType : public Layer +class MeanPooling { public: - //! Create the MeanPoolingType object. - MeanPoolingType(); + //! Create the MeanPooling object. + MeanPooling(); /** * Create the MeanPooling object using the specified number of units. @@ -45,14 +45,11 @@ class MeanPoolingType : public Layer * @param strideHeight Width of the stride operation. * @param floor Set to true to use floor method. */ - MeanPoolingType(const size_t kernelWidth, - const size_t kernelHeight, - const size_t strideWidth = 1, - const size_t strideHeight = 1, - const bool floor = true); - - // TODO: copy constructor / move constructor - MeanPoolingType* Clone() const { return new MeanPoolingType(*this); } + MeanPooling(const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth = 1, + const size_t strideHeight = 1, + const bool floor = true); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -61,7 +58,8 @@ class MeanPoolingType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, using 3rd-order tensors as @@ -72,27 +70,64 @@ class MeanPoolingType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the intput width. + size_t const& InputWidth() const { return inputWidth; } + //! Modify the input width. + size_t& InputWidth() { return inputWidth; } + + //! Get the input height. + size_t const& InputHeight() const { return inputHeight; } + //! Modify the input height. + size_t& InputHeight() { return inputHeight; } + + //! Get the output width. + size_t const& OutputWidth() const { return outputWidth; } + //! Modify the output width. + size_t& OutputWidth() { return outputWidth; } + + //! Get the output height. + size_t const& OutputHeight() const { return outputHeight; } + //! Modify the output height. + size_t& OutputHeight() { return outputHeight; } + + //! Get the input size. + size_t InputSize() const { return inSize; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } //! Get the kernel width. - size_t const& KernelWidth() const { return kernelWidth; } + size_t KernelWidth() const { return kernelWidth; } //! Modify the kernel width. size_t& KernelWidth() { return kernelWidth; } //! Get the kernel height. - size_t const& KernelHeight() const { return kernelHeight; } + size_t KernelHeight() const { return kernelHeight; } //! Modify the kernel height. size_t& KernelHeight() { return kernelHeight; } //! Get the stride width. - size_t const& StrideWidth() const { return strideWidth; } + size_t StrideWidth() const { return strideWidth; } //! Modify the stride width. size_t& StrideWidth() { return strideWidth; } //! Get the stride height. - size_t const& StrideHeight() const { return strideHeight; } + size_t StrideHeight() const { return strideHeight; } //! Modify the stride height. size_t& StrideHeight() { return strideHeight; } @@ -101,40 +136,13 @@ class MeanPoolingType : public Layer //! Modify the value of the rounding operation bool& Floor() { return floor; } - //! Get the size of the output. - const std::vector OutputDimensions() const - { - outputDimensions = this->inputDimensions; + //! Get the value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } - // Compute the size of the output. - if (floor) - { - outputDimensions[0] = std::floor((this->inputDimensions[0] - - (double) kernelWidth) / (double) strideWidth + 1); - outputDimensions[1] = std::floor((this->inputDimensions[1] - - (double) kernelHeight) / (double) strideHeight + 1); - offset = 0; - } - else - { - outputDimensions[0] = std::ceil((this->inputDimensions[0] - - (double) kernelWidth) / (double) strideWidth + 1); - outputDimensions[1] = std::ceil((this->inputDimensions[1] - - (double) kernelHeight) / (double) strideHeight + 1); - offset = 1; - } - - // Higher dimensions are not modified. - for (size_t i = 2; i < this->inputDimensions.size(); ++i) - outputDimensions[i] = this->inputDimensions[i]; - - // Cache input size and output size. - channels = 1; - for (size_t i = 2; i < this->inputDimensions.size(); ++i) - channels *= this->inputDimensions[i]; - - return outputDimensions; - } + //! Get the size of the weights. + size_t WeightSize() const { return 0; } /** * Serialize the layer. @@ -149,7 +157,8 @@ class MeanPoolingType : public Layer * @param input The input to be apply the pooling rule. * @param output The pooled result. */ - void Pooling(const InputType& input, OutputType& output) + template + void Pooling(const arma::Mat& input, arma::Mat& output) { arma::Mat inputPre = input; @@ -165,9 +174,9 @@ class MeanPoolingType : public Layer for (size_t i = 0, rowidx = 0; i < output.n_rows; ++i, rowidx += strideWidth) { - InputType subInput = input( - arma::span(rowidx, rowidx + kernelWidth - 1 - offset), - arma::span(colidx, colidx + kernelHeight - 1 - offset)); + double val = 0.0; + size_t rowEnd = rowidx + kernelWidth - 1; + size_t colEnd = colidx + kernelHeight - 1; if (rowEnd > input.n_rows - 1) rowEnd = input.n_rows - 1; @@ -196,9 +205,10 @@ class MeanPoolingType : public Layer * @param input The input to be apply the unpooling rule. * @param output The pooled result. */ - void Unpooling(const InputType& input, - const OutputType& error, - OutputType& output) + template + void Unpooling(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& output) { // This condition comes by comparing the number of operations involved in the brute // force method and the prefix method. Let the area of error be errorArea and area @@ -211,8 +221,7 @@ class MeanPoolingType : public Layer const bool condition = (error.n_elem * kernelHeight * kernelWidth) > (4 * error.n_elem + 2 * input.n_elem); - OutputType unpooledError; - for (size_t j = 0; j < input.n_cols - cStep; j += cStep) + if (condition) { // If this condition is true then theoritically the prefix sum method of // unpooling is faster. The aim of unpooling is to add @@ -251,8 +260,7 @@ class MeanPoolingType : public Layer { for (size_t i = 0, rowidx = 0; i < input.n_rows; i += strideWidth, ++rowidx) { - // We have to add error(i, j) to output(span(rowidx, rowEnd), - // span(colidx, colEnd)). + // We have to add error(i, j) to output(span(rowidx, rowEnd), span(colidx, colEnd)). // The steps of prefix sum method: // // 1. For each (i, j) perform: @@ -354,21 +362,52 @@ class MeanPoolingType : public Layer //! Rounding operation used. bool floor; - //! Locally-stored number channels. - size_t channels; + //! Locally-stored number of input channels. + size_t inSize; - //! Locally-stored cached output dimensions. - std::vector outputDimensions; + //! Locally-stored number of output channels. + size_t outSize; + + //! Locally-stored input width. + size_t inputWidth; + + //! Locally-stored input height. + size_t inputHeight; + + //! Locally-stored output width. + size_t outputWidth; + + //! Locally-stored output height. + size_t outputHeight; + + //! Locally-stored reset parameter used to initialize the module once. + bool reset; + + //! If true use maximum a posteriori during the forward pass. + bool deterministic; //! Locally-stored number of input units. size_t batchSize; - //! Cached last-seen input. - arma::Cube inputTemp; -}; // class MeanPoolingType + //! Locally-stored output parameter. + arma::cube outputTemp; + + //! Locally-stored transformed input parameter. + arma::cube inputTemp; + + //! Locally-stored transformed output parameter. + arma::cube gTemp; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class MeanPooling -// Standard MeanPooling layer. -typedef MeanPoolingType MeanPooling; } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp new file mode 100644 index 0000000000..ad4a8e6943 --- /dev/null +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -0,0 +1,134 @@ +/** + * @file methods/ann/layer/mean_pooling_impl.hpp + * @author Marcus Edel + * @author Nilay Jain + * + * Implementation of the MeanPooling layer class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_MEAN_POOLING_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_MEAN_POOLING_IMPL_HPP + +// In case it hasn't yet been included. +#include "mean_pooling.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +MeanPooling::MeanPooling() +{ + // Nothing to do here. +} + +template +MeanPooling::MeanPooling( + const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth, + const size_t strideHeight, + const bool floor) : + kernelWidth(kernelWidth), + kernelHeight(kernelHeight), + strideWidth(strideWidth), + strideHeight(strideHeight), + floor(floor), + inSize(0), + outSize(0), + inputWidth(0), + inputHeight(0), + outputWidth(0), + outputHeight(0), + reset(false), + deterministic(false), + batchSize(0) +{ + // Nothing to do here. +} + +template +template +void MeanPooling::Forward( + const arma::Mat& input, arma::Mat& output) +{ + batchSize = input.n_cols; + inSize = input.n_elem / (inputWidth * inputHeight * batchSize); + inputTemp = arma::cube(const_cast&>(input).memptr(), + inputWidth, inputHeight, batchSize * inSize, false, false); + + if (floor) + { + outputWidth = std::floor((inputWidth - + (double) kernelWidth) / (double) strideWidth + 1); + outputHeight = std::floor((inputHeight - + (double) kernelHeight) / (double) strideHeight + 1); + } + else + { + outputWidth = std::ceil((inputWidth - + (double) kernelWidth) / (double) strideWidth + 1); + outputHeight = std::ceil((inputHeight - + (double) kernelHeight) / (double) strideHeight + 1); + } + + outputTemp = arma::zeros >(outputWidth, outputHeight, + batchSize * inSize); + + for (size_t s = 0; s < inputTemp.n_slices; s++) + Pooling(inputTemp.slice(s), outputTemp.slice(s)); + + output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem / batchSize, + batchSize); + + outputWidth = outputTemp.n_rows; + outputHeight = outputTemp.n_cols; + outSize = batchSize * inSize; +} + +template +template +void MeanPooling::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) +{ + arma::cube mappedError = arma::cube(((arma::Mat&) gy).memptr(), + outputWidth, outputHeight, outSize, false, false); + + gTemp = arma::zeros(inputTemp.n_rows, + inputTemp.n_cols, inputTemp.n_slices); + + for (size_t s = 0; s < mappedError.n_slices; s++) + { + Unpooling(inputTemp.slice(s), mappedError.slice(s), gTemp.slice(s)); + } + + g = arma::mat(gTemp.memptr(), gTemp.n_elem / batchSize, batchSize); +} + +template +template +void MeanPooling::serialize( + Archive& ar, + const uint32_t /* version */) +{ + ar(CEREAL_NVP(kernelWidth)); + ar(CEREAL_NVP(kernelHeight)); + ar(CEREAL_NVP(strideWidth)); + ar(CEREAL_NVP(strideHeight)); + ar(CEREAL_NVP(batchSize)); + ar(CEREAL_NVP(floor)); + ar(CEREAL_NVP(inputWidth)); + ar(CEREAL_NVP(inputHeight)); + ar(CEREAL_NVP(outputWidth)); + ar(CEREAL_NVP(outputHeight)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/minibatch_discrimination.hpp b/src/mlpack/methods/ann/layer/minibatch_discrimination.hpp similarity index 61% rename from src/mlpack/methods/ann/layer/not_adapted/minibatch_discrimination.hpp rename to src/mlpack/methods/ann/layer/minibatch_discrimination.hpp index 513021095f..3448f36b5d 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/minibatch_discrimination.hpp +++ b/src/mlpack/methods/ann/layer/minibatch_discrimination.hpp @@ -41,16 +41,16 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class MiniBatchDiscrimination : public Layer +class MiniBatchDiscrimination { public: //! Create the MiniBatchDiscrimination object. @@ -60,16 +60,18 @@ class MiniBatchDiscrimination : public Layer * Create the MiniBatchDiscrimination layer object using the specified * number of units. * + * @param inSize The number of input units. * @param outSize The number of output units. * @param features The number of features to compute for each dimension. */ - MiniBatchDiscrimination(const size_t outSize, + MiniBatchDiscrimination(const size_t inSize, + const size_t outSize, const size_t features); /** * Reset the layer parameter. */ - void SetWeights(typename OutputType::elem_type* weightsPtr); + void Reset(); /** * Ordinary feed-forward pass of a neural network, evaluating the function @@ -78,7 +80,8 @@ class MiniBatchDiscrimination : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed-backward pass of a neural network, calculating the function @@ -89,9 +92,10 @@ class MiniBatchDiscrimination : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activation. @@ -100,27 +104,35 @@ class MiniBatchDiscrimination : public Layer * @param * (error) The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& /* error */, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& /* error */, + arma::Mat& gradient); //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } - const size_t WeightSize() const { return a * b * c; } + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } - const std::vector OutputDimensions() const - { - a = std::accumulate(inputDimensions.begin(), inputDimensions.end(), 0); - std::vector outputDimensions(inputDimensions.size(), 1); - // TODO: not sure if this is right... we just interpret it all as - // one-dimensional. - outputDimensions[0] = a + b; + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } - return outputDimensions; - } + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the shape of the input. size_t InputShape() const @@ -136,22 +148,43 @@ class MiniBatchDiscrimination : public Layer private: //! Locally-stored dimensions of weight. - size_t a, b, c; + size_t A, B, C; //! Locally-stored input batch size. size_t batchSize; - //! Locally-stored weight object. - OutputType weights; + //! Locally-stored temporary features object. + arma::mat tempM; - //! Locally-stored features of input. Cached to avoid recomputation. - InputType M; + //! Locally-stored weight object. + OutputDataType weights; + + //! Locally-stored weight parameters. + OutputDataType weight; + + //! Locally-stored features of input. + arma::cube M; //! Locally-stored delta for features object. - arma::Cube deltaM; + arma::cube deltaM; //! Locally-stored L1 distances between features. - arma::Cube distances; + arma::cube distances; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored temporary delta object. + OutputDataType deltaTemp; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; }; // class MiniBatchDiscrimination } // namespace ann diff --git a/src/mlpack/methods/ann/layer/minibatch_discrimination_impl.hpp b/src/mlpack/methods/ann/layer/minibatch_discrimination_impl.hpp new file mode 100644 index 0000000000..e600b807d4 --- /dev/null +++ b/src/mlpack/methods/ann/layer/minibatch_discrimination_impl.hpp @@ -0,0 +1,147 @@ +/** + * @file methods/ann/layer/minibatch_discrimination_impl.hpp + * @author Saksham Bansal + * + * Implementation of the MiniBatchDiscrimination layer class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_MINIBATCH_DISCRIMINATION_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_MINIBATCH_DISCRIMINATION_IMPL_HPP + +// In case it hasn't yet been included. +#include "minibatch_discrimination.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +MiniBatchDiscrimination::MiniBatchDiscrimination() : + A(0), + B(0), + C(0), + batchSize(0) +{ + // Nothing to do here. +} + +template +MiniBatchDiscrimination::MiniBatchDiscrimination( + const size_t inSize, + const size_t outSize, + const size_t features) : + A(inSize), + B(outSize - inSize), + C(features), + batchSize(0) +{ + weights.set_size(A * B * C, 1); +} + +template +void MiniBatchDiscrimination::Reset() +{ + weight = arma::mat(weights.memptr(), B * C, A, false, false); +} + +template +template +void MiniBatchDiscrimination::Forward( + const arma::Mat& input, arma::Mat& output) +{ + batchSize = input.n_cols; + tempM = weight * input; + M = arma::cube(tempM.memptr(), B, C, batchSize, false, false); + distances.set_size(B, batchSize, batchSize); + output.set_size(B, batchSize); + + for (size_t i = 0; i < M.n_slices; ++i) + { + output.col(i).ones(); + for (size_t j = 0; j < M.n_slices; ++j) + { + if (j < i) + { + output.col(i) += distances.slice(j).col(i); + } + else if (i == j) + { + continue; + } + else + { + distances.slice(i).col(j) = + arma::exp(-arma::sum(abs(M.slice(i) - M.slice(j)), 1)); + output.col(i) += distances.slice(i).col(j); + } + } + } + + output = join_cols(input, output); // (A + B) x batchSize +} + +template +template +void MiniBatchDiscrimination::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) +{ + g = gy.head_rows(A); + arma::Mat gM = gy.tail_rows(B); + deltaM.zeros(B, C, batchSize); + + for (size_t i = 0; i < M.n_slices; ++i) + { + for (size_t j = 0; j < M.n_slices; ++j) + { + if (i == j) + { + continue; + } + arma::mat t = arma::sign(M.slice(i) - M.slice(j)); + t.each_col() %= + distances.slice(std::min(i, j)).col(std::max(i, j)) % gM.col(i); + deltaM.slice(i) -= t; + deltaM.slice(j) += t; + } + } + + deltaTemp = arma::mat(deltaM.memptr(), B * C, batchSize, false, false); + g += weight.t() * deltaTemp; +} + +template +template +void MiniBatchDiscrimination::Gradient( + const arma::Mat& input, + const arma::Mat& /* error */, + arma::Mat& gradient) +{ + gradient = arma::vectorise(deltaTemp * input.t()); +} + +template +template +void MiniBatchDiscrimination::serialize( + Archive& ar, const uint32_t /* version */) +{ + ar(CEREAL_NVP(A)); + ar(CEREAL_NVP(B)); + ar(CEREAL_NVP(C)); + + // This is inefficient, but we have to allocate this memory so that + // WeightSetVisitor gets the right size. + if (cereal::is_loading()) + { + weights.set_size(A * B * C, 1); + } +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/multi_layer.hpp b/src/mlpack/methods/ann/layer/multi_layer.hpp deleted file mode 100644 index 3b9564949b..0000000000 --- a/src/mlpack/methods/ann/layer/multi_layer.hpp +++ /dev/null @@ -1,252 +0,0 @@ -/** - * @file methods/ann/layer/multi_layer.hpp - * @author Ryan Curtin - * - * Base class for neural network layers that are wrappers around other layers. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_MULTI_LAYER_HPP -#define MLPACK_METHODS_ANN_LAYER_MULTI_LAYER_HPP - -#include "../make_alias.hpp" -#include "layer.hpp" - -namespace mlpack { -namespace ann { - -/** - * A "multi-layer" is a layer that is a wrapper around other layers. It passes - * the input through all of its child layers sequentially, returning the output - * from the last layer. - * - * It's likely not very useful to use this layer directly; instead, this layer - * is meant as a base class for use by other layers that must store and use - * multiple layers. - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. - */ -template -class MultiLayer : public Layer -{ - public: - /** - * Create an empty MultiLayer that holds no layers of its own. Be sure to add - * layers with Add() before using! - */ - MultiLayer(); - - //! Copy the given MultiLayer. - MultiLayer(const MultiLayer& other); - //! Take ownership of the layers of the given MultiLayer. - MultiLayer(MultiLayer&& other); - //! Copy the given MultiLayer. - MultiLayer& operator=(const MultiLayer& other); - //! Take ownership of the given MultiLayer. - MultiLayer& operator=(MultiLayer&& other); - - //! Virtual destructor: delete all held layers. - virtual ~MultiLayer() - { - for (size_t i = 0; i < network.size(); ++i) - delete network[i]; - } - - //! Create a copy of the MultiLayer (this is safe for polymorphic use). - virtual MultiLayer* Clone() const { return new MultiLayer(*this); } - - /** - * Perform a forward pass with the given input data. `output` is expected to - * have the correct size (e.g. number of rows equal to `OutputSize()` of the - * last held layer; number of columns equal to `input.n_cols`). - * - * @param input Input data to pass through the MultiLayer. - * @param output Matrix to store output in. - */ - virtual void Forward(const MatType& input, MatType& output); - - /** - * Perform a forward pass with the given input data, but only on a subset of - * the layers in the MultiLayer. `output` is expected to have the correct - * size (e.g. number of rows equal to `OutputSize()` of the last layer to be - * computed; number of columns equal to `input.n_cols`). - * - * @param input Input data to pass through the MultiLayer. - * @param output Matrix to store output in. - * @param start Index of first layer to pass data through. - * @param end Index of last layer to pass data through. - */ - void Forward(const MatType& input, - MatType& output, - const size_t start, - const size_t end); - - /** - * Perform a backward pass with the given data. `gy` is expected to be the - * propagated error from the subsequent layer (or output), `input` is expected - * to be the output from this layer when `Forward()` was called, and `g` will - * store the propagated error from this layer (to be passed to the previous - * layer as `gy`). - * - * It is expected that `g` has the correct size already (e.g., number of rows - * equal to `OutputSize()` of the previous layer, and number of columns equal - * to `input.n_cols`). - * - * This function is expected to be called for the same input data as - * `Forward()` was just called for. - * - * @param input Output of Forward(). - * @param gy Propagated error from next layer. - * @param g Matrix to store propagated error in for previous layer. - */ - virtual void Backward(const MatType& input, - const MatType& gy, - MatType& g); - - /** - * Compute the gradients of each layer. - * - * This function is expected to be called for the same input data as - * `Forward()` and `Backward()` were just called for. That is, `input` here - * should be the same data as `Forward()` was called with. - * - * `gradient` is expected to have the correct size already (e.g., number of - * rows equal to 1, and number of columns equal to `WeightSize()`). - * - * @param input Original input data provided to Forward(). - * @param error Error as computed by `Backward()`. - * @param gradient Matrix to store the gradients in. - */ - virtual void Gradient(const MatType& input, - const MatType& error, - MatType& gradient); - - /** - * Set the weights of the layer to use the memory given as `weightsPtr`. - */ - virtual void SetWeights(typename MatType::elem_type* weightsPtr); - - /** - * Return the number of weights in the MultiLayer. This is the sum of the - * number of weights in each layer. - */ - virtual size_t WeightSize() const; - - /** - * Compute the output dimensions of the MultiLayer using `InputDimensions()`. - * This computes the dimensions of each layer held by the MultiLayer, and the - * output dimensions are set to the output dimensions of the last layer. - */ - virtual void ComputeOutputDimensions(); - - /** - * Compute the loss that should be added to the objective. - */ - virtual double Loss() const; - - /* - * Add a new module to the model. - * - * @param args The layer parameter. - */ - template - void Add(Args... args) - { - network.push_back(new LayerType(args...)); - layerOutputs.push_back(MatType()); - layerDeltas.push_back(MatType()); - layerGradients.push_back(MatType()); - } - - /* - * Add a new module to the model. - * - * @param layer The Layer to be added to the model. - */ - void Add(Layer* layer) - { - network.push_back(layer); - layerOutputs.push_back(MatType()); - layerDeltas.push_back(MatType()); - layerGradients.push_back(MatType()); - } - - //! Get the network (series of layers) held by this MultiLayer. - const std::vector*> Network() const - { - return network; - } - //! Modify the network (series of layers) held by this MultiLayer. Be - //! careful! - std::vector*>& Network() { return network; } - - //! Serialize the MultiLayer. - template - void serialize(Archive& ar, const uint32_t /* version */); - - protected: - /** - * Initialize memory that will be used by each layer for the forward pass, - * assuming that the input will have the given `batchSize`. When `Forward()` - * is called, each internally-held layer will output its results into the - * memory allocated by this function (this is the internal member - * `layerOutputMatrix` and its aliases `layerOutputs`). - */ - void InitializeForwardPassMemory(const size_t batchSize); - - /** - * Initialize memory that will be used by each layer for the backwards pass, - * assuming that the input will have the given `batchSize`. When `Backward()` - * is called, each internally-held layer will output the results of its - * backwards pass into the memory allocated by this function (this is the - * internal member `layerDeltaMatrix` and its aliases `layerDeltas`). - */ - void InitializeBackwardPassMemory(const size_t batchSize); - - /** - * Initialize memory for the gradient pass. This sets the internal aliases - * `layerGradients` appropriately using the memory from the given `gradient`, - * such that each layer will output its gradient (via its `Gradient()` method) - * into the appropriate member of `layerGradients`. - */ - void InitializeGradientPassMemory(MatType& gradient); - - //! The internally-held network. - std::vector*> network; - - // Total number of elements in the input, cached for convenience. - size_t inSize; - // Total number of input elements for *every* layer. - size_t totalInputSize; - // Total number of output elements for *every* layer. - size_t totalOutputSize; - - //! This matrix stores all of the outputs of each layer when Forward() is - //! called. See `InitializeForwardPassMemory()`. - MatType layerOutputMatrix; - //! These are aliases of `layerOutputMatrix` for each layer. - std::vector layerOutputs; - - //! This matrix stores all of the backwards pass results of each layer when - //! Backward() is called. See `InitializeBackwardPassMemory()`. - MatType layerDeltaMatrix; - //! These are aliases of `layerDeltaMatrix` for each layer. - std::vector layerDeltas; - - //! Gradient aliases for each layer. Note that this is *only* valid in the - //! context of `Gradient()`! We have it as a class member to avoid - //! reallocating the `MatType`s each call to `Gradient()`. - std::vector layerGradients; -}; - -} // namespace ann -} // namespace mlpack - -// Include implementation. -#include "multi_layer_impl.hpp" - -#endif diff --git a/src/mlpack/methods/ann/layer/multi_layer_impl.hpp b/src/mlpack/methods/ann/layer/multi_layer_impl.hpp deleted file mode 100644 index 55ab7719a1..0000000000 --- a/src/mlpack/methods/ann/layer/multi_layer_impl.hpp +++ /dev/null @@ -1,414 +0,0 @@ -/** - * @file methods/ann/layer/multi_layer_impl.hpp - * @author Ryan Curtin - * - * Implementation of the base class for neural network layers that are wrappers - * around other layers. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_MULTI_LAYER_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_MULTI_LAYER_IMPL_HPP - -#include "multi_layer.hpp" - -namespace mlpack { -namespace ann { - -template -MultiLayer::MultiLayer() : - inSize(0), - totalInputSize(0), - totalOutputSize(0) -{ - // Nothing to do. -} - -template -MultiLayer::MultiLayer(const MultiLayer& other) : - Layer(other), - inSize(other.inSize), - totalInputSize(other.totalInputSize), - totalOutputSize(other.totalOutputSize), - layerOutputMatrix(other.layerOutputMatrix), - layerDeltaMatrix(other.layerDeltaMatrix) -{ - // Copy each layer. - for (size_t i = 0; i < other.network.size(); ++i) - network.push_back(other.network[i]->Clone()); - - // Ensure that the aliases for layers during passes have the right size. - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - - // layerOutputs, layerDeltas, and layerGradients will be reset the next time - // Forward(), Backward(), or Gradient() is called. -} - -template -MultiLayer::MultiLayer(MultiLayer&& other) : - Layer(other), - network(std::move(other.network)), - inSize(std::move(other.inSize)), - totalInputSize(std::move(other.totalInputSize)), - totalOutputSize(std::move(other.totalOutputSize)), - layerOutputMatrix(std::move(other.layerOutputMatrix)), - layerDeltaMatrix(std::move(other.layerDeltaMatrix)) -{ - // Ensure that the aliases for layers during passes have the right size. - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - - // layerOutputs, layerDeltas, and layerGradients will be reset the next time - // Forward(), Backward(), or Gradient() is called. - - other.layerOutputs.clear(); - other.layerDeltas.clear(); - other.layerGradients.clear(); -} - -template -MultiLayer& MultiLayer::operator=(const MultiLayer& other) -{ - if (this != &other) - { - Layer::operator=(other); - - network.clear(); - layerOutputs.clear(); - layerDeltas.clear(); - layerGradients.clear(); - - inSize = other.inSize; - totalInputSize = other.totalInputSize; - totalOutputSize = other.totalOutputSize; - - layerOutputMatrix = other.layerOutputMatrix; - layerDeltaMatrix = other.layerDeltaMatrix; - - for (size_t i = 0; i < other.network.size(); ++i) - network.push_back(other.network[i]->Clone()); - - // Ensure that the aliases for layers during passes have the right size. - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - } - - return *this; -} - -template -MultiLayer& MultiLayer::operator=(MultiLayer&& other) -{ - if (this != &other) - { - Layer::operator=(other); - - layerOutputs.clear(); - layerDeltas.clear(); - layerGradients.clear(); - - inSize = std::move(other.inSize); - totalInputSize = std::move(other.totalInputSize); - totalOutputSize = std::move(other.totalOutputSize); - - network = std::move(other.network); - - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - - other.layerOutputs.clear(); - other.layerDeltas.clear(); - other.layerGradients.clear(); - } - - return *this; -} - -template -void MultiLayer::Forward( - const MatType& input, MatType& output) -{ - Forward(input, output, 0, network.size() - 1); -} - -template -void MultiLayer::Forward( - const MatType& input, - MatType& output, - const size_t start, - const size_t end) -{ - // Make sure training/testing mode is set right in each layer. - for (size_t i = 0; i < network.size(); ++i) - network[i]->Training() = this->training; - - // Note that we use `output` for the last layer; layerOutputs is only used for - // intermediate values between layers. - if ((end - start) > 0) - { - // Initialize memory for the forward pass (if needed). - InitializeForwardPassMemory(input.n_cols); - - network[start]->Forward(input, layerOutputs[start]); - for (size_t i = start + 1; i < end; ++i) - network[i]->Forward(layerOutputs[i - 1], layerOutputs[i]); - network[end]->Forward(layerOutputs[end - 1], output); - } - else if ((end - start) == 0 && network.size() > 0) - { - network[start]->Forward(input, output); - } - else - { - // Empty network? - output = input; - } -} - -template -void MultiLayer::Backward( - const MatType& input, const MatType& gy, MatType& g) -{ - if (network.size() > 1) - { - // Initialize memory for the backward pass (if needed). - InitializeBackwardPassMemory(input.n_cols); - - network.back()->Backward(input, gy, layerDeltas.back()); - for (size_t i = network.size() - 2; i > 0; --i) - network[i]->Backward(layerOutputs[i], layerDeltas[i + 1], layerDeltas[i]); - network[0]->Backward(layerOutputs[0], layerDeltas[1], g); - } - else if (network.size() == 1) - { - network[0]->Backward(input, gy, g); - } - else - { - // Empty network? - g = input; - } -} - -template -void MultiLayer::Gradient( - const MatType& input, const MatType& error, MatType& gradient) -{ - // We assume gradient has the right size already. - - // Pass gradients through each layer. - if (network.size() > 1) - { - // Initialize memory for the gradient pass (if needed). - InitializeGradientPassMemory(gradient); - - network.front()->Gradient(input, layerDeltas[1], layerGradients.front()); - for (size_t i = 1; i < network.size() - 1; ++i) - { - network[i]->Gradient(layerOutputs[i - 1], layerDeltas[i + 1], - layerGradients[i]); - } - network.back()->Gradient(layerOutputs[network.size() - 2], error, - layerGradients.back()); - } - else if (network.size() == 1) - { - network[0]->Gradient(input, error, gradient); - } - else - { - // Nothing to do if the network is empty... there is no gradient. - } -} - -template -void MultiLayer::SetWeights(typename MatType::elem_type* weightsPtr) -{ - size_t start = 0; - const size_t totalWeightSize = WeightSize(); - for (size_t i = 0; i < network.size(); ++i) - { - const size_t weightSize = network[i]->WeightSize(); - - // Sanity check: ensure we aren't passing memory past the end of the - // parameters. - Log::Assert(start + weightSize <= totalWeightSize, - "FNN::SetLayerMemory(): parameter size does not match total layer " - "weight size!"); - - network[i]->SetWeights(weightsPtr + start); - start += weightSize; - } - - // Technically this check should be unnecessary, but there's nothing wrong - // with a little paranoia... - Log::Assert(start == totalWeightSize, - "FNN::SetLayerMemory(): total layer weight size does not match parameter " - "size!"); -} - -template -size_t MultiLayer::WeightSize() const -{ - // Sum the weights in each layer. - size_t total = 0; - for (size_t i = 0; i < network.size(); ++i) - total += network[i]->WeightSize(); - return total; -} - -template -void MultiLayer::ComputeOutputDimensions() -{ - inSize = 0; - totalInputSize = 0; - totalOutputSize = 0; - - // Propagate the input dimensions forward to the output. - network.front()->InputDimensions() = this->inputDimensions; - inSize = this->inputDimensions[0]; - for (size_t i = 1; i < this->inputDimensions.size(); ++i) - inSize *= this->inputDimensions[i]; - totalInputSize += inSize; - - for (size_t i = 1; i < network.size(); ++i) - { - network[i]->InputDimensions() = network[i - 1]->OutputDimensions(); - size_t layerInputSize = network[i]->InputDimensions()[0]; - for (size_t j = 1; j < network[i]->InputDimensions().size(); ++j) - layerInputSize *= network[i]->InputDimensions()[j]; - - totalInputSize += layerInputSize; - totalOutputSize += layerInputSize; - } - - size_t lastLayerSize = network.back()->OutputDimensions()[0]; - for (size_t i = 1; i < network.back()->OutputDimensions().size(); ++i) - lastLayerSize *= network.back()->OutputDimensions()[i]; - - totalOutputSize += lastLayerSize; - this->outputDimensions = network.back()->OutputDimensions(); -} - -template -double MultiLayer::Loss() const -{ - double loss = 0.0; - for (size_t i = 0; i < network.size(); ++i) - loss += network[i]->Loss(); - - return loss; -} - -template -template -void MultiLayer::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_VECTOR_POINTER(network)); - ar(CEREAL_NVP(inSize)); - ar(CEREAL_NVP(totalInputSize)); - ar(CEREAL_NVP(totalOutputSize)); - - if (Archive::is_loading::value) - { - layerOutputMatrix.clear(); - layerDeltaMatrix.clear(); - layerGradients.clear(); - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - } -} - -template -void MultiLayer::InitializeForwardPassMemory(const size_t batchSize) -{ - // We need to initialize memory to store the output of each layer's Forward() - // call. We'll do this all in one matrix, but, the size of this matrix - // depends on the batch size we are using for computation. We avoid resizing - // layerOutputMatrix down, unless we only need 10% or less of it. - if (batchSize * totalOutputSize > layerOutputMatrix.n_elem || - batchSize * totalOutputSize < - std::floor(0.1 * layerOutputMatrix.n_elem)) - { - // All outputs will be represented by one big block of memory. - layerOutputMatrix = MatType(1, batchSize * totalOutputSize); - } - - // Now, create an alias to the right place for each layer. We assume that - // layerOutputs is already sized correctly (this should be done by Add()). - size_t start = 0; - for (size_t i = 0; i < layerOutputs.size(); ++i) - { - const size_t layerOutputSize = network[i]->OutputSize(); - MakeAlias(layerOutputs[i], layerOutputMatrix.colptr(start), - layerOutputSize, batchSize); - start += batchSize * layerOutputSize; - } -} - -template -void MultiLayer::InitializeBackwardPassMemory( - const size_t batchSize) -{ - // We need to initialize memory to store the output of each layer's Backward() - // call. We do this similarly to InitializeForwardPassMemory(), but we must - // store a matrix to use as the delta for each layer. - if (batchSize * totalInputSize > layerDeltaMatrix.n_elem || - batchSize * totalInputSize < std::floor(0.1 * layerDeltaMatrix.n_elem)) - { - // All deltas will be represented by one big block of memory. - layerDeltaMatrix = MatType(1, batchSize * totalInputSize); - } - - // Now, create an alias to the right place for each layer. We assume that - // layerDeltas is already sized correctly (this should be done by Add()). - size_t start = 0; - for (size_t i = 0; i < layerDeltas.size(); ++i) - { - size_t layerInputSize = 1; - if (i == 0) - { - for (size_t j = 0; j < this->inputDimensions.size(); ++j) - layerInputSize *= this->inputDimensions[j]; - } - else - { - layerInputSize = network[i - 1]->OutputSize(); - } - MakeAlias(layerDeltas[i], layerDeltaMatrix.colptr(start), layerInputSize, - batchSize); - start += batchSize * layerInputSize; - } -} - -template -void MultiLayer::InitializeGradientPassMemory(MatType& gradient) -{ - // We need to initialize memory to store the gradients of each layer. To do - // this, we need to know the weight size of each layer. - size_t gradientStart = 0; - for (size_t i = 0; i < network.size(); ++i) - { - const size_t weightSize = network[i]->WeightSize(); - MakeAlias(layerGradients[i], gradient.memptr() + gradientStart, - weightSize, 1); - gradientStart += weightSize; - } -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/multihead_attention.hpp b/src/mlpack/methods/ann/layer/multihead_attention.hpp similarity index 70% rename from src/mlpack/methods/ann/layer/not_adapted/multihead_attention.hpp rename to src/mlpack/methods/ann/layer/multihead_attention.hpp index e17cb67263..0d7506ea51 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/multihead_attention.hpp +++ b/src/mlpack/methods/ann/layer/multihead_attention.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/multihead_attention.hpp * @author Mrityunjay Tripathi @@ -49,26 +48,25 @@ namespace ann /** Artificial Neural Network. */ { * of shape `(embedDim * tgtSeqLen, batchSize)`. The embeddings are stored * consequently. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam RegularizerType Type of the regularizer to be used. */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, typename RegularizerType = NoRegularizer > -class MultiheadAttentionType : public Layer +class MultiheadAttention { public: /** * Default constructor. */ - MultiheadAttentionType(); + MultiheadAttention(); - // TODO: does srcSeqLen need to be given? /** * Create the MultiheadAttention object using the specified modules. * @@ -76,27 +74,16 @@ class MultiheadAttentionType : public Layer * @param srcSeqLen Source sequence length. * @param embedDim Total dimension of the model. * @param numHeads Number of parallel attention heads. - * @param attnMask Two dimensional Attention Mask. - * @param keyPaddingMask Key Padding Mask. */ - MultiheadAttentionType(const size_t tgtSeqLen, - const size_t srcSeqLen, - const size_t embedDim, - const size_t numHeads, - const InputType& attnmask = InputType(), - const InputType& keyPaddingMask = InputType()); - - //! Clone the MultiheadAttentionType object. This handles polymorphism - //! correctly. - MultiheadAttentionType* Clone() const - { - return new MultiheadAttentionType(*this); - } + MultiheadAttention(const size_t tgtSeqLen, + const size_t srcSeqLen, + const size_t embedDim, + const size_t numHeads); /** * Reset the layer parameters. */ - void SetWeights(typename OutputType::elem_type* weightsPtr); + void Reset(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -105,7 +92,8 @@ class MultiheadAttentionType : public Layer * @param input The query matrix. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -115,9 +103,10 @@ class MultiheadAttentionType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activation. @@ -126,9 +115,10 @@ class MultiheadAttentionType : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the size of the weights. size_t WeightSize() const { return 4 * (embedDim + 1) * embedDim; } @@ -160,27 +150,34 @@ class MultiheadAttentionType : public Layer size_t& NumHeads() { return numHeads; } //! Get the two dimensional Attention Mask. - OutputType const& AttentionMask() const { return attnMask; } + OutputDataType const& AttentionMask() const { return attnMask; } //! Modify the two dimensional Attention Mask. - OutputType& AttentionMask() { return attnMask; } + OutputDataType& AttentionMask() { return attnMask; } //! Get Key Padding Mask. - OutputType const& KeyPaddingMask() const { return keyPaddingMask; } + OutputDataType const& KeyPaddingMask() const { return keyPaddingMask; } //! Modify the Key Padding Mask. - OutputType& KeyPaddingMask() { return keyPaddingMask; } + OutputDataType& KeyPaddingMask() { return keyPaddingMask; } - const size_t WeightSize() const { return (4 * embedDim + 4) * embedDim; } + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } - const std::vector OutputDimensions() const - { - // This returns the output as a 2-dimensional (embedDim * tgtSeqLen) - // matrix. - std::vector outputDimensions(inputDimensions.size(), 1); - outputDimensions[0] = embedDim; - outputDimensions[1] = tgtSeqLen; + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } - return outputDimensions; - } + //! Get the gradient. + OutputDataType const& Gradient() const { return grad; } + //! Modify the gradient. + OutputDataType& Gradient() { return grad; } + + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } size_t InputShape() const { @@ -188,16 +185,16 @@ class MultiheadAttentionType : public Layer } private: - //! Element Type of the output. - typedef typename OutputType::elem_type ElemType; + //! Element Type of the input. + typedef typename OutputDataType::elem_type ElemType; //! Target sequence length. size_t tgtSeqLen; - //! Source sequence length. + //! Source sequence lenght. size_t srcSeqLen; - //! Locally-stored dimensionality of each embedding vector. + //! Locally-stored module output size. size_t embedDim; //! Locally-stored number of parallel attention heads. @@ -207,37 +204,37 @@ class MultiheadAttentionType : public Layer size_t headDim; //! Two dimensional Attention Mask of shape (tgtSeqLen, srcSeqLen). - OutputType attnMask; + OutputDataType attnMask; //! Key Padding Mask. - OutputType keyPaddingMask; + OutputDataType keyPaddingMask; //! Locally-stored weight matrix associated with query. - OutputType queryWt; + OutputDataType queryWt; //! Locally-stored weight matrix associated with key. - OutputType keyWt; + OutputDataType keyWt; //! Locally-stored weight matrix associated with value. - OutputType valueWt; + OutputDataType valueWt; //! Locally-stored weight matrix associated with attnWt. - OutputType outWt; + OutputDataType outWt; //! Locally-stored bias associated with query. - OutputType qBias; + OutputDataType qBias; //! Locally-stored bias associated with key. - OutputType kBias; + OutputDataType kBias; //! Locall-stored bias associated with value. - OutputType vBias; + OutputDataType vBias; //! Locally-stored bias associated with attnWt. - OutputType outBias; + OutputDataType outBias; //! Locally-stored weights parameter. - OutputType weights; + OutputDataType weights; //! Locally-stored projected query matrix over linear layer. arma::Cube qProj; @@ -255,16 +252,20 @@ class MultiheadAttentionType : public Layer arma::Cube attnOut; //! Softmax layer to represent the probabilities of next sequence. - Softmax softmax; + Softmax softmax; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient. + OutputDataType grad; + + //! Locally-stored output parameter. + OutputDataType outputParameter; //! Locally-stored regularizer object. RegularizerType regularizer; }; // class MultiheadAttention - -// Standard MultiheadAttention layer using no regularization. -typedef MultiheadAttentionType - MultiheadAttention; - } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/multihead_attention_impl.hpp b/src/mlpack/methods/ann/layer/multihead_attention_impl.hpp similarity index 79% rename from src/mlpack/methods/ann/layer/not_adapted/multihead_attention_impl.hpp rename to src/mlpack/methods/ann/layer/multihead_attention_impl.hpp index f61d485ab1..3d687d93af 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/multihead_attention_impl.hpp +++ b/src/mlpack/methods/ann/layer/multihead_attention_impl.hpp @@ -21,35 +21,31 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -MultiheadAttentionType:: -MultiheadAttentionType() : +template +MultiheadAttention:: +MultiheadAttention() : tgtSeqLen(0), srcSeqLen(0), embedDim(0), numHeads(0), - headDim(0), - attnMask(InputType()), - keyPaddingMask(InputType()) + headDim(0) { // Nothing to do here. } -template -MultiheadAttentionType:: -MultiheadAttentionType( +template +MultiheadAttention:: +MultiheadAttention( const size_t tgtSeqLen, const size_t srcSeqLen, const size_t embedDim, - const size_t numHeads, - const InputType& attnMask, - const InputType& keyPaddingMask) : + const size_t numHeads) : tgtSeqLen(tgtSeqLen), srcSeqLen(srcSeqLen), embedDim(embedDim), - numHeads(numHeads), - attnMask(attnMask), - keyPaddingMask(keyPaddingMask) + numHeads(numHeads) { if (embedDim % numHeads != 0) { @@ -58,38 +54,41 @@ MultiheadAttentionType( } headDim = embedDim / numHeads; + weights.set_size(WeightSize(), 1); } -template -void MultiheadAttentionType::SetWeights( - typename OutputType::elem_type* weightsPtr) +template +void MultiheadAttention:: +Reset() { - weights = OutputType(weightsPtr, 1, (4 * embedDim + 4) * embedDim, false, - true); + typedef typename arma::Mat MatType; - queryWt = OutputType(weightsPtr, embedDim, embedDim, false, true); - keyWt = OutputType(weightsPtr + embedDim * embedDim, embedDim, embedDim, - false, true); - valueWt = OutputType(weightsPtr + 2 * embedDim * embedDim, embedDim, embedDim, - false, true); - outWt = OutputType(weightsPtr + 3 * embedDim * embedDim, embedDim, embedDim, - false, true); + queryWt = MatType(weights.memptr(), embedDim, embedDim, false, false); + keyWt = MatType(weights.memptr() + embedDim * embedDim, + embedDim, embedDim, false, false); + valueWt = MatType(weights.memptr() + 2 * embedDim * embedDim, + embedDim, embedDim, false, false); + outWt = MatType(weights.memptr() + 3 * embedDim * embedDim, + embedDim, embedDim, false, false); - qBias = OutputType(weightsPtr + 4 * embedDim * embedDim, embedDim, 1, false, - true); - kBias = OutputType(weightsPtr + (4 * embedDim + 1) * embedDim, embedDim, 1, - false, true); - vBias = OutputType(weightsPtr + (4 * embedDim + 2) * embedDim, embedDim, 1, - false, true); - outBias = OutputType(weightsPtr + (4 * embedDim + 3) * embedDim, 1, embedDim, - false, true); + qBias = MatType(weights.memptr() + + 4 * embedDim * embedDim, embedDim, 1, false, false); + kBias = MatType(weights.memptr() + + (4 * embedDim + 1) * embedDim, embedDim, 1, false, false); + vBias = MatType(weights.memptr() + + (4 * embedDim + 2) * embedDim, embedDim, 1, false, false); + outBias = MatType(weights.memptr() + + (4 * embedDim + 3) * embedDim, 1, embedDim, false, false); } -template -void MultiheadAttentionType:: -Forward(const InputType& input, OutputType& output) +template +template +void MultiheadAttention:: +Forward(const arma::Mat& input, arma::Mat& output) { - typedef typename arma::Cube CubeType; + typedef typename arma::Cube CubeType; if (input.n_rows != embedDim * (tgtSeqLen + 2 * srcSeqLen)) { @@ -105,12 +104,12 @@ Forward(const InputType& input, OutputType& output) // The shape of q : (embedDim, tgtSeqLen, batchSize). // The shape of k : (embedDim, srcSeqLen, batchSize). // The shape of v : (embedDim, srcSeqLen, batchSize). - const CubeType q(const_cast(input).memptr(), + const CubeType q(const_cast&>(input).memptr(), embedDim, tgtSeqLen, batchSize, false, false); - const CubeType k(const_cast(input).memptr() + + const CubeType k(const_cast&>(input).memptr() + embedDim * tgtSeqLen * batchSize, embedDim, srcSeqLen, batchSize, false, false); - const CubeType v(const_cast(input).memptr() + + const CubeType v(const_cast&>(input).memptr() + embedDim * (tgtSeqLen + srcSeqLen) * batchSize, embedDim, srcSeqLen, batchSize, false, false); @@ -189,13 +188,15 @@ Forward(const InputType& input, OutputType& output) } } -template -void MultiheadAttentionType:: -Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g) +template +template +void MultiheadAttention:: +Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - typedef typename arma::Cube CubeType; + typedef typename arma::Cube CubeType; if (gy.n_rows != tgtSeqLen * embedDim) { @@ -209,7 +210,7 @@ Backward(const InputType& /* input */, // The shape of gyTemp : (tgtSeqLen, embedDim, batchSize). // We need not split it into n heads now because this is the part when // output were concatenated from n heads. - CubeType gyTemp(const_cast(gy).memptr(), embedDim, + CubeType gyTemp(const_cast&>(gy).memptr(), embedDim, tgtSeqLen, batchSize, true, false); // The shape of gyTemp : (embedDim, tgtSeqLen, batchSize). @@ -279,13 +280,16 @@ Backward(const InputType& /* input */, } } -template -void MultiheadAttentionType:: -Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient) +template +template +void MultiheadAttention:: +Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - typedef typename arma::Cube CubeType; + typedef typename arma::Cube CubeType; + typedef typename arma::Mat MatType; if (input.n_rows != embedDim * (tgtSeqLen + 2 * srcSeqLen)) { @@ -303,16 +307,16 @@ Gradient(const InputType& input, // The shape of gradient : (4 * embedDim * embedDim + 4 * embedDim, 1). gradient.set_size(arma::size(weights)); - const CubeType q(const_cast(input).memptr(), + const CubeType q(const_cast(input).memptr(), embedDim, tgtSeqLen, batchSize, false, false); - const CubeType k(const_cast(input).memptr() + q.n_elem, + const CubeType k(const_cast(input).memptr() + q.n_elem, embedDim, srcSeqLen, batchSize, false, false); - const CubeType v(const_cast(input).memptr() + q.n_elem + k.n_elem, + const CubeType v(const_cast(input).memptr() + q.n_elem + k.n_elem, embedDim, srcSeqLen, batchSize, false, false); // Reshape the propagated error into a cube. // The shape of errorTemp : (embedDim, tgtSeqLen, batchSize). - CubeType errorTemp(const_cast(error).memptr(), embedDim, + CubeType errorTemp(const_cast&>(error).memptr(), embedDim, tgtSeqLen, batchSize, true, false); // Gradient wrt. outBias, i.e. dL/d(outBias). @@ -426,40 +430,22 @@ Gradient(const InputType& input, regularizer.Evaluate(weights, gradient); } -template +template template -void MultiheadAttentionType:: +void MultiheadAttention:: serialize(Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(tgtSeqLen)); ar(CEREAL_NVP(srcSeqLen)); ar(CEREAL_NVP(embedDim)); ar(CEREAL_NVP(numHeads)); ar(CEREAL_NVP(headDim)); - ar(CEREAL_NVP(softmax)); - ar(CEREAL_NVP(regularizer)); - if (Archive::is_loading::value) - { - attnMask.clear(); - keyPaddingMask.clear(); - queryWt.clear(); - keyWt.clear(); - valueWt.clear(); - outWt.clear(); - qBias.clear(); - kBias.clear(); - vBias.clear(); - outBias.clear(); - weights.clear(); - qProj.clear(); - kProj.clear(); - vProj.clear(); - scores.clear(); - attnOut.clear(); - } + // This is inefficient, but we have to allocate this memory so that + // WeightSetVisitor gets the right size. + if (cereal::is_loading()) + weights.set_size(4 * embedDim * (embedDim + 1), 1); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/multiply_constant.hpp b/src/mlpack/methods/ann/layer/multiply_constant.hpp similarity index 64% rename from src/mlpack/methods/ann/layer/not_adapted/multiply_constant.hpp rename to src/mlpack/methods/ann/layer/multiply_constant.hpp index 60e0c72d1a..baa4744c23 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/multiply_constant.hpp +++ b/src/mlpack/methods/ann/layer/multiply_constant.hpp @@ -15,8 +15,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -24,25 +22,22 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the multiply constant layer. The multiply constant layer * multiplies the input by a (non-learnable) constant. * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class MultiplyConstantType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MultiplyConstant { public: - //! Create the MultiplyConstant object. - MultiplyConstantType(const double scalar = 1.0); - - //! Clone the MultiplyConstantType object. This handles polymorphism - //! correctly. - MultiplyConstantType* Clone() const - { - return new MultiplyConstantType(*this); - } + /** + * Create the MultiplyConstant object. + */ + MultiplyConstant(const double scalar = 1.0); //! Copy Constructor. MultiplyConstant(const MultiplyConstant& layer); @@ -63,6 +58,7 @@ class MultiplyConstantType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -73,28 +69,43 @@ class MultiplyConstantType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const DataType& /* input */, const DataType& gy, DataType& g); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the scalar multiplier. double Scalar() const { return scalar; } //! Modify the scalar multiplier. double& Scalar() { return scalar; } - //! Serialize the layer. + //! Get the size of the weights. + size_t WeightSize() const { return 0; } + + /** + * Serialize the layer. + */ template void serialize(Archive& ar, const uint32_t /* version */); private: //! Locally-stored constant scalar value. double scalar; -}; // class MultiplyConstantType -// Convenience typedefs. + //! Locally-stored delta object. + OutputDataType delta; -// Standard MultiplyConstant layer. -typedef MultiplyConstantType MultiplyConstant; + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class MultiplyConstant } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/multiply_constant_impl.hpp b/src/mlpack/methods/ann/layer/multiply_constant_impl.hpp new file mode 100644 index 0000000000..4c02fbd1fa --- /dev/null +++ b/src/mlpack/methods/ann/layer/multiply_constant_impl.hpp @@ -0,0 +1,96 @@ +/** + * @file methods/ann/layer/multiply_constant_impl.hpp + * @author Marcus Edel + * + * Implementation of the MultiplyConstantLayer class, which multiplies the + * input by a (non-learnable) constant. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_MULTIPLY_CONSTANT_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_MULTIPLY_CONSTANT_IMPL_HPP + +// In case it hasn't yet been included. +#include "multiply_constant.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +MultiplyConstant::MultiplyConstant( + const double scalar) : scalar(scalar) +{ + // Nothing to do here. +} + +template +MultiplyConstant::MultiplyConstant( + const MultiplyConstant& layer) : + scalar(layer.scalar) +{ + // Nothing to do here. +} + +template +MultiplyConstant::MultiplyConstant( + MultiplyConstant&& layer) : + scalar(std::move(layer.scalar)) +{ + // Nothing to do here. +} + +template +MultiplyConstant& +MultiplyConstant::operator=( + const MultiplyConstant& layer) +{ + if (this != &layer) + { + scalar = layer.scalar; + } + return *this; +} + +template +MultiplyConstant& +MultiplyConstant::operator=( + MultiplyConstant&& layer) +{ + if (this != &layer) + { + scalar = std::move(layer.scalar); + } + return *this; +} + +template +template +void MultiplyConstant::Forward( + const InputType& input, OutputType& output) +{ + output = input * scalar; +} + +template +template +void MultiplyConstant::Backward( + const DataType& /* input */, const DataType& gy, DataType& g) +{ + g = gy * scalar; +} + +template +template +void MultiplyConstant::serialize( + Archive& ar, const uint32_t /* version */) +{ + ar(CEREAL_NVP(scalar)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/multiply_merge.hpp b/src/mlpack/methods/ann/layer/multiply_merge.hpp similarity index 51% rename from src/mlpack/methods/ann/layer/not_adapted/multiply_merge.hpp rename to src/mlpack/methods/ann/layer/multiply_merge.hpp index b690871063..5c3d9ba6c0 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/multiply_merge.hpp +++ b/src/mlpack/methods/ann/layer/multiply_merge.hpp @@ -15,9 +15,9 @@ #include -// #include "../visitor/delete_visitor.hpp" -// #include "../visitor/delta_visitor.hpp" -// #include "../visitor/output_parameter_visitor.hpp" +#include "../visitor/delete_visitor.hpp" +#include "../visitor/delta_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" #include "layer_types.hpp" @@ -28,16 +28,18 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the MultiplyMerge module class. The MultiplyMerge class * multiplies the output of various modules element-wise. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). + * @tparam CustomLayers Additional custom layers that can be added. */ template< - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename... CustomLayers > -class MultiplyMergeType : public MultiLayer +class MultiplyMerge { public: /** @@ -46,7 +48,7 @@ class MultiplyMergeType : public MultiLayer * @param model Expose all the network modules. * @param run Call the Forward/Backward method before the output is merged. */ - MultiplyMergeType(const bool model = false, const bool run = true); + MultiplyMerge(const bool model = false, const bool run = true); //! Copy Constructor. MultiplyMerge(const MultiplyMerge& layer); @@ -61,10 +63,7 @@ class MultiplyMergeType : public MultiLayer MultiplyMerge& operator=(MultiplyMerge&& layer); //! Destructor to release allocated memory. - ~MultiplyMergeType(); - - //! Clone the MultiplyMergeType object. This handles polymorphism correctly. - MultiplyMergeType* Clone() const { return new MultiplyMergeType(*this); } + ~MultiplyMerge(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -73,6 +72,7 @@ class MultiplyMergeType : public MultiLayer * @param * (input) Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& /* input */, OutputType& output); /** @@ -84,9 +84,10 @@ class MultiplyMergeType : public MultiLayer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -95,14 +96,56 @@ class MultiplyMergeType : public MultiLayer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); + + /* + * Add a new module to the model. + * + * @param args The layer parameter. + */ + template + void Add(Args... args) { network.push_back(new LayerType(args...)); } + + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Return the model modules. + std::vector >& Model() + { + if (model) + { + return network; + } + + return empty; + } //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } //! Get the size of the weights. size_t WeightSize() const { return 0; } @@ -114,6 +157,9 @@ class MultiplyMergeType : public MultiLayer void serialize(Archive& ar, const uint32_t /* version */); private: + //! Parameter which indicates if the modules should be exposed. + bool model; + //! Parameter which indicates if the Forward/Backward method should be called //! before merging the output. bool run; @@ -121,12 +167,33 @@ class MultiplyMergeType : public MultiLayer //! We need this to know whether we should delete the layer in the destructor. bool ownsLayer; - //! Locally-stored weight object. - OutputType weights; -}; // class MultiplyMergeType + //! Locally-stored network modules. + std::vector > network; -// Standard MultiplyMerge layer. -typedef MultiplyMergeType MultiplyMerge; + //! Locally-stored empty list of modules. + std::vector > empty; + + //! Locally-stored delete visitor module object. + DeleteVisitor deleteVisitor; + + //! Locally-stored output parameter visitor module object. + OutputParameterVisitor outputParameterVisitor; + + //! Locally-stored delta visitor module object. + DeltaVisitor deltaVisitor; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored weight object. + OutputDataType weights; +}; // class MultiplyMerge } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp new file mode 100644 index 0000000000..29cd111482 --- /dev/null +++ b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp @@ -0,0 +1,190 @@ +/** + * @file methods/ann/layer/multiply_merge_impl.hpp + * @author Haritha Nair + * + * Definition of the MultiplyMerge module which multiplies the output of the + * given modules element-wise. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_MULTIPLY_MERGE_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_MULTIPLY_MERGE_IMPL_HPP + +// In case it hasn't yet been included. +#include "multiply_merge.hpp" + +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +MultiplyMerge::MultiplyMerge( + const bool model, const bool run) : + model(model), run(run), ownsLayer(!model) +{ + // Nothing to do here. +} + +template +MultiplyMerge::MultiplyMerge( + const MultiplyMerge& layer) : + model(layer.model), + run(layer.run), + ownsLayer(layer.ownsLayer), + network(layer.network), + weights(layer.weights) +{ + // Nothing to do here. +} + +template +MultiplyMerge::MultiplyMerge( + MultiplyMerge&& layer) : + model(std::move(layer.model)), + run(std::move(layer.run)), + ownsLayer(std::move(layer.ownsLayer)), + network(std::move(layer.network)), + weights(std::move(layer.weights)) +{ + // Nothing to do here. +} + +template +MultiplyMerge& +MultiplyMerge::operator=( + const MultiplyMerge& layer) +{ + if (this != &layer) + { + model = layer.model; + run = layer.run; + ownsLayer = layer.ownsLayer; + network = layer.network; + weights = layer.weights; + } + return *this; +} + +template +MultiplyMerge& +MultiplyMerge::operator=( + MultiplyMerge&& layer) +{ + if (this != &layer) + { + model = std::move(layer.model); + run = std::move(layer.run); + ownsLayer = std::move(layer.ownsLayer); + network = std::move(layer.network); + weights = std::move(layer.weights); + } + return *this; +} + +template +MultiplyMerge::~MultiplyMerge() +{ + if (ownsLayer) + { + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deleteVisitor)); + } +} + +template +template +void MultiplyMerge::Forward( + const InputType& input, OutputType& output) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network[i])), + network[i]); + } + } + + output = boost::apply_visitor(outputParameterVisitor, network.front()); + for (size_t i = 1; i < network.size(); ++i) + { + output %= boost::apply_visitor(outputParameterVisitor, network[i]); + } +} + +template +template +void MultiplyMerge::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[i]), gy, + boost::apply_visitor(deltaVisitor, network[i])), network[i]); + } + + g = boost::apply_visitor(deltaVisitor, network[0]); + for (size_t i = 1; i < network.size(); ++i) + { + g += boost::apply_visitor(deltaVisitor, network[i]); + } + } + else + g = gy; +} + +template +template +void MultiplyMerge::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */ ) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(GradientVisitor(input, error), network[i]); + } + } +} + +template +template +void MultiplyMerge::serialize( + Archive& ar, const uint32_t /* version */) +{ + // Be sure to clear other layers before loading. + if (cereal::is_loading()) + network.clear(); + + ar(CEREAL_VECTOR_VARIANT_POINTER(network)); + ar(CEREAL_NVP(model)); + ar(CEREAL_NVP(run)); + ar(CEREAL_NVP(ownsLayer)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/nearest_interpolation.hpp b/src/mlpack/methods/ann/layer/nearest_interpolation.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/nearest_interpolation.hpp rename to src/mlpack/methods/ann/layer/nearest_interpolation.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/nearest_interpolation_impl.hpp b/src/mlpack/methods/ann/layer/nearest_interpolation_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/nearest_interpolation_impl.hpp rename to src/mlpack/methods/ann/layer/nearest_interpolation_impl.hpp diff --git a/src/mlpack/methods/ann/layer/noisylinear.hpp b/src/mlpack/methods/ann/layer/noisylinear.hpp index feac41131f..993f80725f 100644 --- a/src/mlpack/methods/ann/layer/noisylinear.hpp +++ b/src/mlpack/methods/ann/layer/noisylinear.hpp @@ -14,8 +14,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -23,42 +21,55 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the NoisyLinear layer class. It represents a single * layer of a neural network, with parametric noise added to its weights. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class NoisyLinearType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class NoisyLinear { public: + //! Create the NoisyLinear object. + NoisyLinear(); + /** * Create the NoisyLinear layer object using the specified number of units. * + * @param inSize The number of input units. * @param outSize The number of output units. */ - NoisyLinearType(const size_t outSize = 0); + NoisyLinear(const size_t inSize, + const size_t outSize); - //! Clone the NoisyLinearType object. This handles polymorphism correctly. - NoisyLinearType* Clone() const { return new NoisyLinearType(*this); } + //! Copy constructor. + NoisyLinear(const NoisyLinear&); - // Virtual destructor. - virtual ~NoisyLinearType() { } + //! Move constructor. + NoisyLinear(NoisyLinear&&); - //! Copy the given NoisyLinear layer (but not weights). - NoisyLinearType(const NoisyLinearType& other); - //! Take ownership of the given NoisyLinear layer (but not weights). - NoisyLinearType(NoisyLinearType&& other); - //! Copy the given NoisyLinear layer (but not weights). - NoisyLinearType& operator=(const NoisyLinearType& other); - //! Take ownership of the given NoisyLinear layer (but not weights). - NoisyLinearType& operator=(NoisyLinearType&& other); + //! Operator= copy constructor. + NoisyLinear& operator=(const NoisyLinear& layer); - //! Reset the layer parameter. - void SetWeights(typename MatType::elem_type* weightsPtr); + //! Operator= move constructor. + NoisyLinear& operator=(NoisyLinear&& layer); - //! Reset the noise parameters (epsilons). + /* + * Reset the layer parameter. + */ + void Reset(); + + /* + * Reset the noise parameters(epsilons). + */ void ResetNoise(); - //! Reset the values of layer parameters (factorized gaussian noise). + /* + * Reset the values of layer parameters (factorized gaussian noise). + */ void ResetParameters(); /** @@ -68,7 +79,8 @@ class NoisyLinearType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -79,80 +91,117 @@ class NoisyLinearType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, - const MatType& gy, - MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); - /** + /* * Calculate the gradient using the output delta and the input activation. * * @param input The input parameter used for calculating the gradient. * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const MatType& input, - const MatType& error, - MatType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - MatType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - MatType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the input size. + size_t InputSize() const { return inSize; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the shape of the input. + size_t InputShape() const + { + return inSize; + } + //! Modify the bias weights of the layer. - MatType& Bias() { return bias; } + arma::mat& Bias() { return bias; } - //! Compute the number of parameters in the layer. + //! Get size of weights. size_t WeightSize() const { return (outSize * inSize + outSize) * 2; } - - //! Compute the output dimensions of the layer given `InputDimensions()`. - void ComputeOutputDimensions(); - - //! Serialize the layer. + /** + * Serialize the layer + */ template void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored number of input units. + size_t inSize; + //! Locally-stored number of output units. size_t outSize; - //! Locally stored number of input units. - size_t inSize; - //! Locally-stored weight object. - MatType weights; + OutputDataType weights; //! Locally-stored weight parameters. - MatType weight; + OutputDataType weight; //! Locally-stored weight-mean parameters. - MatType weightMu; + OutputDataType weightMu; //! Locally-stored weight-standard-deviation parameters. - MatType weightSigma; + OutputDataType weightSigma; //! Locally-stored weight-epsilon parameters. - MatType weightEpsilon; + OutputDataType weightEpsilon; //! Locally-stored bias parameters. - MatType bias; + OutputDataType bias; //! Locally-stored bias-mean parameters. - MatType biasMu; + OutputDataType biasMu; //! Locally-stored bias-standard-deviation parameters. - MatType biasSigma; + OutputDataType biasSigma; //! Locally-stored bias-epsilon parameters. - MatType biasEpsilon; + OutputDataType biasEpsilon; -}; // class NoisyLinearType + //! Locally-stored delta object. + OutputDataType delta; -// Convenience typedefs. + //! Locally-stored gradient object. + OutputDataType gradient; -// Standard noisy linear layer. -typedef NoisyLinearType NoisyLinear; + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class NoisyLinear } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/noisylinear_impl.hpp b/src/mlpack/methods/ann/layer/noisylinear_impl.hpp index 4df8685283..c7a37b5863 100644 --- a/src/mlpack/methods/ann/layer/noisylinear_impl.hpp +++ b/src/mlpack/methods/ann/layer/noisylinear_impl.hpp @@ -18,92 +18,107 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -NoisyLinearType::NoisyLinearType(const size_t outSize) : - Layer(), - outSize(outSize), - inSize(0) +template +NoisyLinear::NoisyLinear() : + inSize(0), + outSize(0) { // Nothing to do here. } -template -NoisyLinearType::NoisyLinearType(const NoisyLinearType& other) : - Layer(other), - outSize(other.outSize), - inSize(other.inSize) +template +NoisyLinear::NoisyLinear( + const NoisyLinear& layer) : + inSize(layer.inSize), + outSize(layer.outSize), + weights(layer.weights) { - // Nothing to do. + Reset(); } -template -NoisyLinearType::NoisyLinearType(NoisyLinearType&& other) : - Layer(std::move(other)), - outSize(std::move(other.outSize)), - inSize(std::move(other.inSize)) +template +NoisyLinear::NoisyLinear( + const size_t inSize, + const size_t outSize) : + inSize(inSize), + outSize(outSize) { - // Nothing to do. + weights.set_size(WeightSize(), 1); + weightEpsilon.set_size(outSize, inSize); + biasEpsilon.set_size(outSize, 1); } -template -NoisyLinearType& -NoisyLinearType::operator=(const NoisyLinearType& other) +template +NoisyLinear::NoisyLinear( + NoisyLinear&& layer) : + inSize(std::move(layer.inSize)), + outSize(std::move(layer.outSize)), + weights(std::move(layer.weights)) { - if (&other != this) + layer.inSize = 0; + layer.outSize = 0; + layer.weights = nullptr; + Reset(); +} + +template +NoisyLinear& +NoisyLinear::operator=(const NoisyLinear& layer) +{ + if (this != &layer) { - Layer::operator=(other); - outSize = other.outSize; - inSize = other.inSize; + inSize = layer.inSize; + outSize = layer.outSize; + weights = layer.weights; + Reset(); } - return *this; } -template -NoisyLinearType& -NoisyLinearType::operator=(NoisyLinearType&& other) +template +NoisyLinear& +NoisyLinear::operator=(NoisyLinear&& layer) { - if (&other != this) + if (this != &layer) { - Layer::operator=(std::move(other)); - outSize = std::move(other.outSize); - inSize = std::move(other.inSize); + inSize = std::move(layer.inSize); + layer.inSize = 0; + outSize = std::move(layer.outSize); + layer.outSize = 0; + weights = std::move(layer.weights); + layer.weights = nullptr; + Reset(); } - return *this; } -template -void NoisyLinearType::SetWeights( - typename MatType::elem_type* weightsPtr) +template +void NoisyLinear::Reset() { - MakeAlias(weights, weightsPtr, 1, (outSize * inSize + outSize) * 2); - - MakeAlias(weightMu, weightsPtr, outSize, inSize); - MakeAlias(biasMu, weightsPtr + weightMu.n_elem, outSize, 1); - MakeAlias(weightSigma, weightsPtr + weightMu.n_elem + biasMu.n_elem, outSize, - inSize); - MakeAlias(biasSigma, weightsPtr + weightMu.n_elem * 2 + biasMu.n_elem, - outSize, 1); - + weightMu = arma::mat(weights.memptr(), + outSize, inSize, false, false); + biasMu = arma::mat(weights.memptr() + weightMu.n_elem, + outSize, 1, false, false); + weightSigma = arma::mat(weights.memptr() + weightMu.n_elem + biasMu.n_elem, + outSize, inSize, false, false); + biasSigma = arma::mat(weights.memptr() + weightMu.n_elem * 2 + biasMu.n_elem, + outSize, 1, false, false); this->ResetNoise(); } -template -void NoisyLinearType::ResetNoise() +template +void NoisyLinear::ResetNoise() { - MatType epsilonIn = arma::randn(inSize, 1); + arma::mat epsilonIn = arma::randn(inSize, 1); epsilonIn = arma::sign(epsilonIn) % arma::sqrt(arma::abs(epsilonIn)); - - MatType epsilonOut = arma::randn(outSize, 1); + arma::mat epsilonOut = arma::randn(outSize, 1); epsilonOut = arma::sign(epsilonOut) % arma::sqrt(arma::abs(epsilonOut)); - weightEpsilon = epsilonOut * epsilonIn.t(); biasEpsilon = epsilonOut; } -template -void NoisyLinearType::ResetParameters() +template +void NoisyLinear::ResetParameters() { const double muRange = 1 / std::sqrt(inSize); weightMu.randu(); @@ -114,8 +129,10 @@ void NoisyLinearType::ResetParameters() biasSigma.fill(0.5 / std::sqrt(outSize)); } -template -void NoisyLinearType::Forward(const MatType& input, MatType& output) +template +template +void NoisyLinear::Forward( + const arma::Mat& input, arma::Mat& output) { weight = weightMu + weightSigma % weightEpsilon; bias = biasMu + biasSigma % biasEpsilon; @@ -123,19 +140,23 @@ void NoisyLinearType::Forward(const MatType& input, MatType& output) output.each_col() += bias; } -template -void NoisyLinearType::Backward( - const MatType& /* input */, const MatType& gy, MatType& g) +template +template +void NoisyLinear::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { g = weight.t() * gy; } -template -void NoisyLinearType::Gradient( - const MatType& input, const MatType& error, MatType& gradient) +template +template +void NoisyLinear::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { // Locally stored to prevent multiplication twice. - MatType weightGrad = error * input.t(); + arma::mat weightGrad = error * input.t(); // Gradients for mu values. gradient.rows(0, weight.n_elem - 1) = arma::vectorise(weightGrad); @@ -149,34 +170,18 @@ void NoisyLinearType::Gradient( = arma::sum(error, 1) % biasEpsilon; } -template -void NoisyLinearType::ComputeOutputDimensions() -{ - inSize = this->inputDimensions[0]; - for (size_t i = 1; i < this->inputDimensions.size(); ++i) - inSize *= this->inputDimensions[i]; - - this->outputDimensions = std::vector(this->inputDimensions.size(), - 1); - - // The NoisyLinear layer flattens its output. - this->outputDimensions[0] = outSize; -} - -template +template template -void NoisyLinearType::serialize( +void NoisyLinear::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(outSize)); ar(CEREAL_NVP(inSize)); + ar(CEREAL_NVP(outSize)); + // This is inefficient, but we have to allocate this memory so that + // WeightSetVisitor gets the right size. if (cereal::is_loading()) - { - ResetNoise(); - } + weights.set_size((outSize * inSize + outSize) * 2, 1); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/README.md b/src/mlpack/methods/ann/layer/not_adapted/README.md deleted file mode 100644 index ffe867395b..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Layers in this directory were written with the old boost::visitor interface. In -[#2777](https://github.com/mlpack/mlpack/pull/2777), we adapted each layer to -use inheritance instead. However, time did not permit the adaptation of all -layers, and so remaining layers that have not yet been adapted are in this -directory. - -The intention is that we will work our way through layers in this directory, -updating them to the new interface and re-enabling tests for them in separate, -follow-up PRs. If you'd like to help out, you are more than welcome to! diff --git a/src/mlpack/methods/ann/layer/not_adapted/add_merge_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/add_merge_impl.hpp deleted file mode 100644 index f1c71ae3ad..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/add_merge_impl.hpp +++ /dev/null @@ -1,145 +0,0 @@ -/** - * @file methods/ann/layer/add_merge_impl.hpp - * @author Marcus Edel - * - * Definition of the AddMerge module which accumulates the output of the given - * modules. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP - -// In case it hasn't yet been included. -#include "add_merge.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -AddMerge::AddMerge( - const bool run) : - run(run), ownsLayers(true) -{ - // Nothing to do here. -} - -template -AddMerge::AddMerge( - const bool run, const bool ownsLayers) : - run(run), ownsLayers(ownsLayers) -{ - // Nothing to do here. -} - -template -AddMerge::~AddMerge() -{ - -} - -template -void AddMerge::Forward( - const InputType& input, OutputType& output) -{ - this->InitializeForwardPassMemory(); - - if (run) - { - for (size_t i = 0; i < this->network.size(); ++i) - { - this->network[i]->Forward(input, this->layerOutputs[i]); - } - } - - output = this->layerOutputs.front(); - for (size_t i = 1; i < this->network.size(); ++i) - { - output += this->layerOutputs[i]; - } -} - -template -void AddMerge::Backward( - const InputType& /* input */, - const OutputType& gy, - OutputType& g) -{ - this->InitializeBackwardPassMemory(); - - if (run) - { - for (size_t i = 0; i < this->network.size(); ++i) - { - this->network[i]->Backward(this->layerOutputs[i], gy, - this->layerDeltas[i]); - } - - g = this->layerDeltas[0]; - for (size_t i = 1; i < this->network.size(); ++i) - { - g += this->layerDeltas[i]; - } - } - else - { - g = gy; - } -} - -template -void AddMerge::Backward( - const InputType& /* input */, - const OutputType& gy, - OutputType& g, - const size_t index) -{ - this->network[index]->Backward(this->layerOutputs[index], gy, g); -} - -template -void AddMerge::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) -{ - if (run) - { - size_t start = 0; - for (size_t i = 0; i < this->network.size(); ++i) - { - this->network[i]->Gradient(input, error, OutputType(gradient.colptr(start), - 1, this->network[i]->WeightSize(), false, true)); - start += this->network[i]->WeightSize(); - } - } -} - -template -void AddMerge::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient, - const size_t index) -{ - this->network[index]->Gradient(input, error, gradient); -} - -template -template -void AddMerge::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(run)); - ar(CEREAL_NVP(ownsLayers)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/concat.hpp b/src/mlpack/methods/ann/layer/not_adapted/concat.hpp deleted file mode 100644 index 1d99ff60c9..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/concat.hpp +++ /dev/null @@ -1,222 +0,0 @@ -/** - * @file methods/ann/layer/concat.hpp - * @author Marcus Edel - * @author Mehul Kumar Nirala - * - * Definition of the Concat class, which acts as a concatenation container. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_CONCAT_HPP -#define MLPACK_METHODS_ANN_LAYER_CONCAT_HPP - -#include - -#include "layer.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -/** - * Implementation of the Concat class. The Concat class works as a - * feed-forward fully connected network container which plugs various layers - * together. - * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - */ -template < - typename InputType = arma::mat, - typename OutputType = arma::mat -> -class ConcatType : public MultiLayer -{ - public: - /** - * Create the Concat object using the specified parameters. - * - * @param run Call the Forward/Backward method before the output is merged. - */ - ConcatType(const bool run = true); - - /** - * Create the Concat object, specifying a particular axis on which the layer - * outputs should be concatenated. - * - * @param axis Concat axis. - * @param run Call the Forward/Backward method before the output is merged. - */ - ConcatType(const size_t axis, const bool run = true); - - /** - * Destroy the layers held by the model. - */ - ~ConcatType(); - - //! Clone the ConcatType object. This handles polymorphism correctly. - ConcatType* Clone() const { return new ConcatType(*this); } - - /** - * Ordinary feed forward pass of a neural network, evaluating the function - * f(x) by propagating the activity forward through f. - * - * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. - */ - void Forward(const InputType& input, OutputType& output); - - /** - * Ordinary feed backward pass of a neural network, using 3rd-order tensors as - * input, calculating the function f(x) by propagating x backwards through f. - * Using the results from the feed forward pass. - * - * @param * (input) The propagated input activation. - * @param gy The backpropagated error. - * @param g The calculated gradient. - */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); - - /** - * This is the overload of Backward() that runs only a specific layer with - * the given input. - * - * @param * (input) The propagated input activation. - * @param gy The backpropagated error. - * @param g The calculated gradient. - * @param index The index of the layer to run. - */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g, - const size_t index); - - /** - * Calculate the gradient using the output delta and the input activation. - * - * @param input The input parameter used for calculating the gradient. - * @param error The calculated error. - * @param gradient The calculated gradient. - */ - void Gradient(const InputType& /* input */, - const OutputType& error, - OutputType& /* gradient */); - - /** - * This is the overload of Gradient() that runs a specific layer with the - * given input. - * - * @param input The input parameter used for calculating the gradient. - * @param error The calculated error. - * @param gradient The calculated gradient. - * @param The index of the layer to run. - */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient, - const size_t index); - - //! Get the value of run parameter. - bool Run() const { return run; } - //! Modify the value of run parameter. - bool& Run() { return run; } - - //! Get the axis of concatenation. - const size_t& ConcatAxis() const { return axis; } - - //! Get the size of the weight matrix. - size_t WeightSize() const { return 0; } - - void ComputeOutputDimensions() - { - // The input is sent to every layer. - for (size_t i = 0; i < network.size(); ++i) - { - network[i]->InputDimensions() = this->inputDimensions; - network[i]->ComputeOutputDimensions(); - } - - // If the user did not specify an axis, we will use the last one. - // Otherwise, we must sanity check to ensure that the axis we are - // concatenating along is valid. - if (!useAxis) - { - axis = this->inputDimensions.size() - 1; - } - else if (axis >= this->inputDimensions.size()) - { - std::ostringstream oss; - oss << "Concat::ComputeOutputDimensions(): cannot concatenate outputs " - << "along axis " << axis << " when input only has " - << this->inputDimensions.size() << " axes!"; - throw std::invalid_argument(oss.str()); - } - - // Now, we concatenate the output along a specific axis. - this->outputDimensions = std::vector(this->inputDimensions.size(), - 0); - for (size_t i = 0; i < this->inputDimensions.size(); ++i) - { - if (i == axis) - { - // Accumulate output size along this axis for each layer output. - for (size_t n = 0; n < this->network.size(); ++n) - { - this->outputDimensions[i] += this->network[n]->OutputDimensions()[i]; - } - } - else - { - // Ensure that the output size is the same along this axis. - const size_t axisDim = this->network[0]->OutputDimensions()[i]; - for (size_t n = 1; n < this->network.size(); ++n) - { - const size_t axisDim2 = this->network[n]->OutputDimensions()[i]; - if (axisDim != axisDim2) - { - std::ostringstream oss; - oss << "Concat::ComputeOutputDimensions(): cannot concatenate " - << "outputs along axis " << axis << "; held layer " << n - << " has output size " << axisDim2 << " along axis " << i - << ", but the first held layer has output size " << axisDim - << "! All layers must have identical output size in any " - << "axis other than the concatenated axis."; - throw std::invalid_argument(oss.str()); - } - } - - this->outputDimensions[i] = axisDim; - } - } - } - - /** - * Serialize the layer - */ - template - void serialize(Archive& ar, const uint32_t /* version */); - - private: - //! Parameter which indicates the axis of concatenation. - size_t axis; - - //! Parameter which indicates whether to use the axis of concatenation. - bool useAxis; -}; // class ConcatType. - -// Standard Concat layer. -typedef ConcatType Concat; - -} // namespace ann -} // namespace mlpack - -// Include implementation. -#include "concat_impl.hpp" - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/concat_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/concat_impl.hpp deleted file mode 100644 index 7e231b5f81..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/concat_impl.hpp +++ /dev/null @@ -1,267 +0,0 @@ -/** - * @file methods/ann/layer/concat_impl.hpp - * @author Marcus Edel - * @author Mehul Kumar Nirala - * - * Implementation of the Concat class, which acts as a concatenation contain. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_CONCAT_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_CONCAT_IMPL_HPP - -// In case it hasn't yet been included. -#include "concat.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -ConcatType::ConcatType( - const bool run) : - axis(0), - useAxis(false) -{ - // Nothing to do. -} - -template -ConcatType::ConcatType( - const size_t axis, - const bool run) : - axis(axis), - useAxis(true) -{ - // Nothing to do. -} - -template -ConcatType::~ConcatType() -{ - // Clear memory. - for (size_t i = 0; i < this->network.size(); ++i) - delete this->network[i]; -} - -template -void ConcatType::Forward( - const InputType& input, OutputType& output) -{ - this->InitializeForwardPassMemory(); - - // Pass the input through all the layers in the network. - for (size_t i = 0; i < this->network.size(); ++i) - { - this->network[i]->Forward(input, this->layerOutputs[i]); - } - - // Now concatenate the outputs along the correct axis. - // We can actually use Armadillo to do this for us---we will treat the axis of - // interest as "columns", any axes that come before the axis of interest as - // 'flattened slices', and any axes that come after the axis of interest as - // 'flattened rows'. As a result, we will only have to do join_cols() to - // produce the right result. - // - // Note that we will have one "extra" axis in addition to - // this->outputDimensions.size(); that is the batch size (represented as the - // number of columns in `input`). - - size_t slices = (axis == 0) ? input.n_cols : - std::accumulate(this->outputDimensions.begin(), - this->outputDimensions.begin() + axis, 0) + input.n_cols; - size_t rows = (axis == this->outputDimensions.size() - 1) ? 1 : - std::accumulate(this->outputDimensions.begin() + axis + 1, - this->outputDimensions.end(), 0); - - std::vector> layerOutputAliases; - for (size_t i = 0; i < this->layerOutputs.size(); ++i) - { - layerOutputAliases.emplace_back(arma::Cube( - this->layerOutputs[i].memptr(), rows, - this->network[i]->OutputDimensions()[axis], slices, false, true); - } - - arma::Cube output(output.memptr(), rows, - this->outputDimensions[axis], slices, false, true); - - // Now get the columns from each output. - size_t startCol = 0; - for (size_t i = 0; i < layerOutputAliases.size(); ++i) - { - const size_t cols = layerOutputAliases[i].n_cols; - output.cols(startCol, startCol + cols - 1) = layerOutputAliases[i]; - startCol += cols; - } -} - -template -void ConcatType::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) -{ - this->InitializeBackwardPassMemory(); - - // Just like the forward pass, we can treat our inputs as a cube, but here we - // have to distribute the correct parts of `gy` to the layers. - - size_t slices = (axis == 0) ? gy.n_cols : - std::accumulate(this->outputDimensions.begin(), - this->outputDimensions.begin() + axis, 0) + gy.n_cols; - size_t rows = (axis == this->outputDimensions.size() - 1) ? 1 : - std::accumulate(this->outputDimensions.begin() + axis + 1, - this->outputDimensions.end(), 0); - - arma::Cube gyTmp(gy.memptr(), rows, - this->outputDimensions[axis], slices, false, true); - - size_t startCol = 0; - for (size_t i = 0; i < this->network.size(); ++i) - { - const size_t cols = this->network[i]->OutputDimensions()[axis]; - // TODO: is delta size correct? - // TODO: no copy! - OutputType delta = gyTmp.cols(startCol, startCol + cols - 1); - // TODO: consider batch size correctly - delta.reshape( ... ); - this->network[i]->Backward(this->layerOutputs[i], delta, - this->layerDeltas[i]); - - startCol += cols; - } - - g = this->layerDeltas[0]; - for (size_t i = 1; i < this->network.size(); ++i) - { - g += this->layerDeltas[i]; - } -} - -template -void ConcatType::Backward( - const InputType& /* input */, - const OutputType& gy, - OutputType& g, - const size_t index) -{ - // We only intend to perform a backward pass on one layer. - // Thus, we need to extract the parts of gy that correspond to the desired - // layer (specified by `index`). - - size_t slices = (axis == 0) ? gy.n_cols : - std::accumulate(this->outputDimensions.begin(), - this->outputDimensions.begin() + axis, 0) + gy.n_cols; - size_t rows = (axis == this->outputDimensions.size() - 1) ? 1 : - std::accumulate(this->outputDimensions.begin() + axis + 1, - this->outputDimensions.end(), 0); - - arma::Cube gyTmp(gy.memptr(), rows, - this->outputDimensions[axis], slices, false, true); - - size_t startCol = 0; - for (size_t i = 0; i < index; ++i) - { - startCol += this->network[i]->OutputDimensions()[axis]; - } - - // TODO: no copy! - const size_t cols = this->network[index]->OutputDimensions()[axis]; - OutputType delta = gyTmp.cols(startCol, startCol + cols - 1); - delta.reshape( ... ); - - this->network[index]->Backward(this->layerOutputs[index], delta, g); -} - -template -void ConcatType::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) -{ - // Just like the forward pass, we can treat our inputs as a cube, but here we - // have to distribute the correct parts of `gy` to the layers. - - size_t slices = (axis == 0) ? input.n_cols : - std::accumulate(this->outputDimensions.begin(), - this->outputDimensions.begin() + axis, 0) + input.n_cols; - size_t rows = (axis == this->outputDimensions.size() - 1) ? 1 : - std::accumulate(this->outputDimensions.begin() + axis + 1, - this->outputDimensions.end(), 0); - - arma::Cube errorTmp(error.memptr(), rows, - this->outputDimensions[axis], slices, false, true); - - size_t startCol = 0; - size_t startParam = 0; - for (size_t i = 0; i < this->network.size(); ++i) - { - const size_t cols = this->network[i]->OutputDimensions()[axis]; - const size_t params = this->network[i]->WeightSize(); - - OutputType err = errorTmp.cols(startCol, startCol + cols - 1); - err.reshape(input.n_cols, err.n_elem / input.n_cols); - // TODO: what about layerGradients? - OutputType gradientAlias(gradient.colptr(startParam, 1, params, false, - true); - this->network[i]->Gradient(input, err, gradientAlias); - - startCol += cols; - startParam += params; - } -} - -// TODO: adapt -template -void ConcatType::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient, - const size_t index) -{ - // Just like the forward pass, we can treat our inputs as a cube, but here we - // have to distribute the correct parts of `gy` to the layers. - - size_t slices = (axis == 0) ? input.n_cols : - std::accumulate(this->outputDimensions.begin(), - this->outputDimensions.begin() + axis, 0) + input.n_cols; - size_t rows = (axis == this->outputDimensions.size() - 1) ? 1 : - std::accumulate(this->outputDimensions.begin() + axis + 1, - this->outputDimensions.end(), 0); - - arma::Cube errorTmp(error.memptr(), rows, - this->outputDimensions[axis], slices, false, true); - - size_t startCol = 0; - size_t startParam = 0; - for (size_t i = 0; i < index; ++i) - { - startCol += this->network[i]->OutputDimensions()[axis]; - startParam += this->network[i]->WeightSize(); - } - - const size_t cols = this->network[index]->OutputDimensions()[axis]; - const size_t params = this->network[index]->WeightSize(); - - // TODO: no copy! - OutputType err = errorTmp.cols(startCol, startCol + cols - 1); - err.reshape(input.n_cols, err.n_elem / input.n_cols); - OutputType gradientAlias(gradient.memptr(), 1, params, false, true); - this->network[index]->Gradient(input, err, gradientAlias); -} - -template -template -void ConcatType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(axis)); - ar(CEREAL_NVP(useAxis)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/constant_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/constant_impl.hpp deleted file mode 100644 index 601b5c2e76..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/constant_impl.hpp +++ /dev/null @@ -1,118 +0,0 @@ -/** - * @file methods/ann/layer/constant_impl.hpp - * @author Marcus Edel - * - * Implementation of the Constant class, which outputs a constant value given - * any input. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_CONSTANT_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_CONSTANT_IMPL_HPP - -// In case it hasn't yet been included. -#include "constant.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -ConstantType::ConstantType() : - outSize(0) -{ - // Nothing to do. -} - -template -ConstantType::ConstantType( - const size_t outSize, - const double scalar) : - outSize(outSize) -{ - constantOutput = OutputType(outSize, 1); - constantOutput.fill(scalar); -} - -template -ConstantType::ConstantType( - const ConstantType& other) : - outSize(other.outSize), - constantOutput(other.constantOutput) -{ - // Nothing else to do. -} - -template -ConstantType::ConstantType( - ConstantType&& other) : - outSize(other.outSize), - constantOutput(std::move(other.constantOutput)) -{ - other.outSize = 1; - other.constantOutput = OutputType(other.outSize, 1); -} - -template -ConstantType& -ConstantType::operator=( - const ConstantType& other) -{ - if (this != &other) - { - outSize = other.outSize; - constantOutput = other.constantOutput; - } - - return *this; -} - -template -ConstantType& -ConstantType::operator=( - ConstantType&& other) -{ - if (this != *other) - { - outSize = other.outSize; - constantOutput = std::move(other.constantOutput); - - other.outSize = 1; - other.constantOutput = OutputType(other.outSize, 1); - } - - return *this; -} - -template -void ConstantType::Forward( - const InputType& input, OutputType& output) -{ - output = constantOutput; -} - -template -void ConstantType::Backward( - const InputType& /* input */, const OutputType& /* gy */, OutputType& g) -{ - g.zeros(); -} - -template -template -void ConstantType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(constantOutput)); - if (Archive::is_loading::value) - outSize = constantOutput.n_elem; -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/gru_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/gru_impl.hpp deleted file mode 100644 index 2e7fb208c2..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/gru_impl.hpp +++ /dev/null @@ -1,369 +0,0 @@ -/** - * @file methods/ann/layer/gru_impl.hpp - * @author Sumedh Ghaisas - * - * Implementation of the GRU class, which implements a gru network - * layer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_GRU_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_GRU_IMPL_HPP - -// In case it hasn't yet been included. -#include "gru.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -GRU::GRU() -{ - // Nothing to do here. -} - -template -GRU::GRU( - const size_t inSize, - const size_t outSize, - const size_t rho) : - inSize(inSize), - outSize(outSize), - rho(rho), - batchSize(1), - forwardStep(0), - backwardStep(0), - gradientStep(0) -{ - // Input specific linear layers(for zt, rt, ot). - input2GateModule = new LinearType(inSize, 3 * outSize); - - // Previous output gates (for zt and rt). - output2GateModule = new LinearNoBiasType(outSize, - 2 * outSize); - - // Previous output gate for ot. - outputHidden2GateModule = new LinearNoBiasType(outSize, - outSize); - - network.push_back(input2GateModule); - network.push_back(output2GateModule); - network.push_back(outputHidden2GateModule); - - inputGateModule = new SigmoidLayer(); - forgetGateModule = new SigmoidLayer(); - hiddenStateModule = new TanHLayer(); - - network.push_back(inputGateModule); - network.push_back(hiddenStateModule); - network.push_back(forgetGateModule); - - prevError = arma::zeros(3 * outSize, batchSize); - - allZeros = arma::zeros(outSize, batchSize); - - outParameter.emplace_back(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true); - - prevOutput = outParameter.begin(); - backIterator = outParameter.end(); - gradIterator = outParameter.end(); -} - -template -void GRU::Forward( - const InputType& input, OutputType& output) -{ - if (input.n_cols != batchSize) - { - batchSize = input.n_cols; - prevError.resize(3 * outSize, batchSize); - allZeros.zeros(outSize, batchSize); - // Batch size better not change during an iteration... - if (outParameter.size() > 1) - { - Log::Fatal << "GRU<>::Forward(): batch size cannot change during a " - << "forward pass!" << std::endl; - } - - outParameter.clear(); - outParameter.emplace_back(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true); - - prevOutput = outParameter.begin(); - backIterator = outParameter.end(); - gradIterator = outParameter.end(); - } - - // Process the input linearly(zt, rt, ot). - input2GateModule->Forward(input, input2GateModule->OutputParameter()); - - // Process the output(zt, rt) linearly. - output2GateModule->Forward(*prevOutput, output2GateModule->OutputParameter()); - - // Merge the outputs(zt and rt). - output = input2GateModule->OutputParameter().submat(0, 0, 2 * outSize - 1, - batchSize - 1) + output2GateModule->OutputParameter(); - - // Pass the first outSize through inputGate(it). - inputGateModule->Forward(output.submat( 0, 0, 1 * outSize - 1, batchSize - 1), - inputGateModule->OutputParameter()); - - // Pass the second through forgetGate. - forgetGateModule->Forward(output.submat( 1 * outSize, 0, 2 * outSize - 1, - batchSize - 1), forgetGateModule->OutputParameter()); - - OutputType modInput = forgetGateModule->OutputParameter() % *prevOutput; - - // Pass that through the outputHidden2GateModule. - outputHidden2GateModule->Forward(modInput, - outputHidden2GateModule->OutputParameter()); - - // Merge for ot. - OutputType outputH = input2GateModule->OutputParameter().submat(2 * outSize, - 0, 3 * outSize - 1, batchSize - 1) + - outputHidden2GateModule->OutputParameter(); - - // Pass it through hiddenGate. - hiddenStateModule->ForwardVisitor(outputH, - hiddenStateModule->OutputParameter()); - - // Update the output (nextOutput): cmul1 + cmul2 - // Where cmul1 is input gate * prevOutput and - // cmul2 is (1 - input gate) * hidden gate. - output = (inputGateModule->OutputParameter() - % (*prevOutput - hiddenStateModule->OutputParameter())) + - hiddenStateModule->OutputParameter(); - - forwardStep++; - if (forwardStep == rho) - { - forwardStep = 0; - if (this->training) - { - outParameter.emplace_back(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true); - prevOutput = --outParameter.end(); - } - else - { - *prevOutput = arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true); - } - } - else if (this->training) - { - outParameter.push_back(output); - prevOutput = --outParameter.end(); - } - else - { - if (forwardStep == 1) - { - outParameter.clear(); - outParameter.push_back(output); - - prevOutput = outParameter.begin(); - } - else - { - *prevOutput = output; - } - } -} - -template -void GRU::Backward( - const InputType& input, const OutputType& gy, OutputType& g) -{ - if (input.n_cols != batchSize) - { - batchSize = input.n_cols; - prevError.resize(3 * outSize, batchSize); - allZeros.zeros(outSize, batchSize); - // Batch size better not change during an iteration... - if (outParameter.size() > 1) - { - Log::Fatal << "GRU<>::Forward(): batch size cannot change during a " - << "forward pass!" << std::endl; - } - - outParameter.clear(); - outParameter.emplace_back(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true); - - prevOutput = outParameter.begin(); - backIterator = outParameter.end(); - gradIterator = outParameter.end(); - } - - OutputType gyLocal; - if ((outParameter.size() - backwardStep - 1) % rho != 0 && backwardStep != 0) - { - gyLocal = gy + output2GateModule->Delta(); - } - else - { - gyLocal = OutputType(((OutputType&) gy).memptr(), gy.n_rows, gy.n_cols, - false, false); - } - - if (backIterator == outParameter.end()) - { - backIterator = --(--outParameter.end()); - } - - // Delta zt. - OutputType dZt = gyLocal % (*backIterator - - hiddenStateModule->OutputParameter()); - - // Delta ot. - OutputType dOt = gyLocal % (arma::ones(outSize, batchSize) - - inputGateModule->OutputParameter()); - - // Delta of input gate. - inputGateModule->Backward(inputGateModule->OutputParameter(), dZt, - inputGateModule->Delta()); - - // Delta of hidden gate. - hiddenStateModule->Backward(hiddenStateModule->OutputParameter(), dOt, - hiddenStateModule->Delta()); - - // Delta of outputHidden2GateModule. - outputHidden2GateModule->Backward(outputHidden2GateModule->OutputParameter(), - hiddenStateModule->Delta(), outputHidden2GateModule->Delta()); - - // Delta rt. - OutputType dRt = outputHidden2GateModule->Delta() % *backIterator; - - // Delta of forget gate. - forgetGateModule->Backward(forgetGateModule->OutputParameter(), dRt, - forgetGateModule->Delta()); - - // Put delta zt. - prevError.submat(0, 0, 1 * outSize - 1, batchSize - 1) = - inputGateModule->Delta(); - - // Put delta rt. - prevError.submat(1 * outSize, 0, 2 * outSize - 1, batchSize - 1) = - forgetGateModule->Delta(); - - // Put delta ot. - prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1) = - hiddenStateModule->Delta(); - - // Get delta ht - 1 for input gate and forget gate. - OutputType prevErrorSubview = prevError.submat(0, 0, 2 * outSize - 1, - batchSize - 1); - output2GateModule->Backward(input2GateModule->OutputParameter(), - prevErrorSubview, output2GateModule->Delta()); - - // Add delta ht - 1 from hidden state. - output2GateModule->Delta() += outputHidden2GateModule->Delta() % - forgetGateModule->OutputParameter(); - - // Add delta ht - 1 from ht. - output2GateModule->Delta() += gyLocal % inputGateModule->OutputParameter(); - - // Get delta input. - input2GateModule->Backward(input2GateModule->OutputParameter(), prevError, - input2GateModule->Delta()); - - backwardStep++; - backIterator--; - - g = input2GateModule->Delta(); -} - -template -void GRU::Gradient( - const InputType& input, - const OutputType& /* error */, - OutputType& /* gradient */) -{ - if (input.n_cols != batchSize) - { - batchSize = input.n_cols; - prevError.resize(3 * outSize, batchSize); - allZeros.zeros(outSize, batchSize); - // Batch size better not change during an iteration... - if (outParameter.size() > 1) - { - Log::Fatal << "GRU<>::Forward(): batch size cannot change during a " - << "forward pass!" << std::endl; - } - - outParameter.clear(); - outParameter.emplace_back(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true); - - prevOutput = outParameter.begin(); - backIterator = outParameter.end(); - gradIterator = outParameter.end(); - } - - if (gradIterator == outParameter.end()) - { - gradIterator = --(--outParameter.end()); - } - - input2GateModule->Gradient(input, prevError, input2GateModule->Gradient()); - - output2GateModule->Gradient(*gradIterator, - prevError.submat(0, 0, 2 * outSize - 1, batchSize - 1), - output2GateModule->Gradient()); - - outputHidden2GateModule->Gradient( - *gradIterator % forgetGateModule->OutputParameter(), - prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchSize - 1), - outputHidden2GateModule->Gradient()); - - gradIterator--; -} - -template -void GRU::ResetCell(const size_t /* size */) -{ - outParameter.clear(); - outParameter.emplace_back(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true); - - prevOutput = outParameter.begin(); - backIterator = outParameter.end(); - gradIterator = outParameter.end(); - - forwardStep = 0; - backwardStep = 0; -} - -template -template -void GRU::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - // If necessary, clean memory from the old model. - // TODO: CEREAL_POINTER() should clean memory automatically... - - ar(CEREAL_NVP(inSize)); - ar(CEREAL_NVP(outSize)); - ar(CEREAL_NVP(rho)); - - ar(CEREAL_NVP(weights)); - - ar(CEREAL_POINTER(input2GateModule)); - ar(CEREAL_POINTER(output2GateModule)); - ar(CEREAL_POINTER(outputHidden2GateModule)); - ar(CEREAL_POINTER(inputGateModule)); - ar(CEREAL_POINTER(forgetGateModule)); - ar(CEREAL_POINTER(hiddenStateModule)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/highway.hpp b/src/mlpack/methods/ann/layer/not_adapted/highway.hpp deleted file mode 100644 index 749149e1c3..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/highway.hpp +++ /dev/null @@ -1,157 +0,0 @@ -// Temporarily drop. -/** - * @file methods/ann/layer/highway.hpp - * @author Konstantin Sidorov - * @author Saksham Bansal - * - * Definition of the Highway layer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP -#define MLPACK_METHODS_ANN_LAYER_HIGHWAY_HPP - -#include - -#include "layer.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -/** - * Implementation of the Highway layer. The Highway class can vary its behavior - * between that of feed-forward fully connected network container and that - * of a layer which simply passes its inputs through depending on the transform - * gate. Note that the size of the input and output matrices of this class - * should be equal. - * - * For more information, refer the following paper. - * - * @code - * @article{Srivastava2015, - * author = {Rupesh Kumar Srivastava, Klaus Greff, Jurgen Schmidhuber}, - * title = {Training Very Deep Networks}, - * journal = {Advances in Neural Information Processing Systems}, - * year = {2015}, - * url = {https://arxiv.org/abs/1507.06228}, - * } - * @endcode - * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - */ -template < - typename InputType = arma::mat, - typename OutputType = arma::mat -> -class HighwayType : public MultiLayer -{ - public: - //! Create the HighwayType object. - HighwayType(); - - //! Destroy the HighwayType object. - virtual ~HighwayType(); - - //! Clone the HighwayType object. This handles polymorphism correctly. - HighwayType* Clone() const { return new HighwayType(*this); } - - //! Copy the given HighwayType (but not weights). - HighwayType(const HighwayType& other); - //! Take ownership of the given HighwayType (but not weights). - HighwayType(HighwayType&& other); - //! Copy the given HighwayType (but not weights). - HighwayType& operator=(const HighwayType& other); - //! Take ownership of the given HighwayType (but not weights). - HighwayType& operator=(HighwayType&& other); - - void SetWeights(typename OutputType::elem_type* weightsPtr); - - /** - * Ordinary feed-forward pass of a neural network, evaluating the function - * f(x) by propagating the activity forward through f. - * - * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. - */ - void Forward(const InputType& input, OutputType& output); - - /** - * Ordinary feed-backward pass of a neural network, calculating the function - * f(x) by propagating x backwards through f. Using the results from the - * feed-forward pass. - * - * @param * (input) The propagated input activation. - * @param gy The backpropagated error. - * @param g The calculated gradient. - */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); - - /** - * Calculate the gradient using the output delta and the input activation. - * - * @param input The input parameter used for calculating the gradient. - * @param error The calculated error. - * @param gradient The calculated gradient. - */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); - - //! Get the parameters. - OutputType const& Parameters() const { return weights; } - //! Modify the parameters. - OutputType& Parameters() { return weights; } - - //! Get the number of trainable weights. - size_t WeightSize() const - { - size_t result = this->totalInputSize * (this->totalInputSize + 1); - for (size_t i = 0; i < this->network.size(); ++i) - result += this->network[i]->WeightSize(); - return result; - } - - /** - * Serialize the layer. - */ - template - void serialize(Archive& ar, const uint32_t /* version */); - - private: - //! Locally-stored weight object. - OutputType weights; - - //! Weights for transformation of output. - OutputType transformWeight; - - //! Bias for transformation of output. - OutputType transformBias; - - //! Locally-stored transform gate parameters. - OutputType transformGate; - - //! Locally-stored transform gate activation. - OutputType transformGateActivation; - - //! Locally-stored transform gate error. - OutputType transformGateError; -}; // class HighwayType - -// Standard Highway layer. -typedef HighwayType Highway; - -} // namespace ann -} // namespace mlpack - -// Include implementation. -#include "highway_impl.hpp" - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/highway_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/highway_impl.hpp deleted file mode 100644 index 0b019a8486..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/highway_impl.hpp +++ /dev/null @@ -1,194 +0,0 @@ -/** - * @file methods/ann/layer/highway_impl.hpp - * @author Konstantin Sidorov - * @author Saksham Bansal - * - * Implementation of Highway layer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_HIGHWAY_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_HIGHWAY_IMPL_HPP - -// In case it hasn't yet been included. -#include "highway.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -HighwayType::HighwayType() -{ - // Nothing to do here. - // TODO: how do we add the child layers ?? (read paper ...) -} - -template -HighwayType::~HighwayType() -{ - // Nothing to do. -} - -template -HighwayType::HighwayType(const HighwayType& other) : - MultiLayer(other) -{ - // Nothing to do. -} - -template -HighwayType::HighwayType(HighwayType&& other) : - MultiLayer(std::move(other)) -{ - // Nothing to do. -} - -template -HighwayType& -HighwayType::operator=(const HighwayType& other) -{ - if (&other == this) - { - MultiLayer::operator=(other); - } - - return *this; -} - -template -HighwayType& -HighwayType::operator=(HighwayType&& other) -{ - if (&other == this) - { - MultiLayer::operator=(std::move(other)); - } - - return *this; -} - -template -void HighwayType::SetWeights( - typename OutputType::elem_type* weightsPtr) -{ - transformWeight = OutputType(weightsPtr, this->inSize, - this->inSize, false, false); - transformBias = OutputType(weightsPtr + transformWeight.n_elem, - this->inSize, 1, false, false); - - size_t start = transformWeight.n_elem + transformBias.n_elem; - for (size_t i = 0; i < this->network.size(); ++i) - { - this->network[i]->SetWeights(weightsPtr + start); - start += this->network[i]->WeightSize(); - } -} - -template -void HighwayType::Forward( - const InputType& input, OutputType& output) -{ - this->InitializeForwardPassMemory(input.n_cols); - - this->network.front()->Forward(input, this->layerOutputs.front()); - - for (size_t i = 1; i < this->network.size(); ++i) - { - this->network[i]->Forward(this->layerOutputs[i - 1], this->layerOutputs[i]); - } - - output = this->layerOutputs.back(); // TODO: can this be cleaned up? - - // TODO: move to ComputeOutputDimensions() - if (arma::size(output) != arma::size(input)) - { - Log::Fatal << "The sizes of the output and input matrices of the Highway" - << " network should be equal. Please examine the network layers."; - } - - transformGate = transformWeight * input; - transformGate.each_col() += transformBias; - transformGateActivation = 1.0 /(1 + arma::exp(-transformGate)); - output = (this->layerOutputs.back() % transformGateActivation) + - (input % (1 - transformGateActivation)); -} - -template -void HighwayType::Backward( - const InputType& input, - const OutputType& gy, - OutputType& g) -{ - this->InitializeBackwardPassMemory(input.n_cols); - - OutputType gyTransform = gy % transformGateActivation; - this->network.back()->Backward(this->layerOutputs.back(), gyTransform, - this->layerDeltas.back()); - - for (size_t i = 2; i < this->network.size() + 1; ++i) - { - this->network[this->network.size() - i]->Backward( - this->layerOutputs[this->network.size() - i], - this->layerDeltas[this->network.size() - i + 1], - this->layerDeltas[this->network.size() - i]); - } - - transformGateError = gy % (gy - input) % - transformGateActivation % (1.0 - transformGateActivation); - g = this->layerDeltas.front() + (transformWeight.t() * transformGateError) + - (gy % (1 - transformGateActivation)); -} - -template -void HighwayType::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) -{ - // Create an alias for the gradient that only refers to the elements in the - // network itself. - OutputType layerGradient(gradient.memptr() + (this->inSize * - (this->inSize + 1)), 1, gradient.n_elem - (this->inSize * - (this->inSize + 1)), false, true); - this->InitializeGradientPassMemory(layerGradient); - - OutputType errorTransform = error % transformGateActivation; - this->network.back()->Gradient( - this->layerOutputs[this->network.size() - 2], - errorTransform, - this->layerGradients[this->network.size() - 1]); - - for (size_t i = 2; i < this->network.size(); ++i) - { - this->network[this->network.size() - i]->Gradient( - this->layerOutputs[this->network.size() - i - 1], - this->layerDeltas[this->network.size() - i], - this->layerGradients[this->network.size() - i]); - } - - this->network.front()->Gradient( - input, - this->layerDeltas[1], - this->layerGradients.front()); - - gradient.submat(0, 0, transformWeight.n_elem - 1, 0) = arma::vectorise( - transformGateError * input.t()); - gradient.submat(transformWeight.n_elem, 0, transformWeight.n_elem + - transformBias.n_elem - 1, 0) = arma::sum(transformGateError, 1); -} - -template -template -void HighwayType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/mean_pooling_impl.hpp deleted file mode 100644 index 845f264c72..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/mean_pooling_impl.hpp +++ /dev/null @@ -1,108 +0,0 @@ -/** - * @file methods/ann/layer/mean_pooling_impl.hpp - * @author Marcus Edel - * @author Nilay Jain - * - * Implementation of the MeanPooling layer class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_MEAN_POOLING_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_MEAN_POOLING_IMPL_HPP - -// In case it hasn't yet been included. -#include "mean_pooling.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -MeanPoolingType::MeanPoolingType() -{ - // Nothing to do here. -} - -template -MeanPoolingType::MeanPoolingType( - const size_t kernelWidth, - const size_t kernelHeight, - const size_t strideWidth, - const size_t strideHeight, - const bool floor) : - kernelWidth(kernelWidth), - kernelHeight(kernelHeight), - strideWidth(strideWidth), - strideHeight(strideHeight), - floor(floor), - channels(0), - offset(0), - batchSize(0) -{ - // Nothing to do here. -} - -template -void MeanPoolingType::Forward( - const InputType& input, OutputType& output) -{ - batchSize = input.n_cols; - inputTemp = arma::Cube( - const_cast(input).memptr(), this->inputDimensions[0], - this->inputDimensions[1], batchSize * channels, false, false); - - arma::Cube outputTemp(output.memptr(), - outputDimensions[0], outputDimensions[1], batchSize * channels, false, - true); - - for (size_t s = 0; s < inputTemp.n_slices; s++) - Pooling(inputTemp.slice(s), outputTemp.slice(s)); -} - -template -void MeanPoolingType::Backward( - const InputType& /* input */, - const OutputType& gy, - OutputType& g) -{ - arma::Cube mappedError( - ((OutputType&) gy).memptr(), outputDimensions[0], outputDimensions[1], - batchSize * channels, false, true); - - arma::Cube gTemp(g.memptr(), - this->inputDimensions[0], this->inputDimensions[1], channels * batchSize, - false, true); - - for (size_t s = 0; s < mappedError.n_slices; s++) - { - Unpooling(inputTemp.slice(s), mappedError.slice(s), gTemp.slice(s)); - } -} - -template -template -void MeanPoolingType::serialize( - Archive& ar, - const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(kernelWidth)); - ar(CEREAL_NVP(kernelHeight)); - ar(CEREAL_NVP(strideWidth)); - ar(CEREAL_NVP(strideHeight)); - ar(CEREAL_NVP(batchSize)); - ar(CEREAL_NVP(floor)); - ar(CEREAL_NVP(outputDimensions)); - ar(CEREAL_NVP(offset)); - - if (Archive::is_loading::value) - inputTemp.clear(); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/minibatch_discrimination_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/minibatch_discrimination_impl.hpp deleted file mode 100644 index ea098a4c25..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/minibatch_discrimination_impl.hpp +++ /dev/null @@ -1,138 +0,0 @@ -/** - * @file methods/ann/layer/minibatch_discrimination_impl.hpp - * @author Saksham Bansal - * - * Implementation of the MiniBatchDiscrimination layer class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_MINIBATCH_DISCRIMINATION_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_MINIBATCH_DISCRIMINATION_IMPL_HPP - -// In case it hasn't yet been included. -#include "minibatch_discrimination.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -MiniBatchDiscrimination::MiniBatchDiscrimination() : - A(0), - B(0), - C(0), - batchSize(0) -{ - // Nothing to do here. -} - -template -MiniBatchDiscrimination::MiniBatchDiscrimination( - const size_t outSize, - const size_t features) : - a(0), // This will be set when OutputDimensions() is called. - b(outSize - inSize), - c(features), - batchSize(0) -{ - // Nothing to do. -} - -template -void MiniBatchDiscrimination::SetWeights( - typename OutputType::elem_type* weightsPtr) -{ - weights = OutputType(weightsPtr, b * c, a, false, false); -} - -template -void MiniBatchDiscrimination::Forward( - const InputType& input, OutputType& output) -{ - batchSize = input.n_cols; - M = weight * input; - arma::Cube cubeM(M.memptr(), b, c, batchSize, - false, false); - distances.set_size(b, batchSize, batchSize); - - for (size_t i = 0; i < cubeM.n_slices; ++i) - { - output.col(i).subvec(0, a - 1) = input.col(i); - output.col(i).subvec(a, output.n_rows - 1).ones(); - for (size_t j = 0; j < cubeM.n_slices; ++j) - { - if (j < i) - { - output.col(i).subvec(a, output.n_rows - 1) += distances.slice(j).col(i); - } - else if (i == j) - { - continue; - } - else - { - distances.slice(i).col(j) = - arma::exp(-arma::sum(abs(cubeM.slice(i) - cubeM.slice(j)), 1)); - output.col(i) += distances.slice(i).col(j); - } - } - } -} - -template -void MiniBatchDiscrimination::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) -{ - g = gy.head_rows(a); - OutputType gM = gy.tail_rows(B); - deltaM.zeros(b, c, batchSize); - - for (size_t i = 0; i < M.n_slices; ++i) - { - for (size_t j = 0; j < M.n_slices; ++j) - { - if (i == j) - { - continue; - } - InputType t = arma::sign(M.slice(i) - M.slice(j)); - t.each_col() %= - distances.slice(std::min(i, j)).col(std::max(i, j)) % gM.col(i); - deltaM.slice(i) -= t; - deltaM.slice(j) += t; - } - } - - OutputType deltaTemp(deltaM.memptr(), b * c, batchSize, false, true); - g += weight.t() * deltaTemp; -} - -template -void MiniBatchDiscrimination::Gradient( - const InputType& input, - const OutputType& /* error */, - OutputType& gradient) -{ - gradient = arma::vectorise(deltaTemp * input.t()); -} - -template -template -void MiniBatchDiscrimination::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(a)); - ar(CEREAL_NVP(b)); - ar(CEREAL_NVP(c)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/multiply_constant_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/multiply_constant_impl.hpp deleted file mode 100644 index 61ae392dea..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/multiply_constant_impl.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/** - * @file methods/ann/layer/multiply_constant_impl.hpp - * @author Marcus Edel - * - * Implementation of the MultiplyConstantLayer class, which multiplies the - * input by a (non-learnable) constant. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_MULTIPLY_CONSTANT_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_MULTIPLY_CONSTANT_IMPL_HPP - -// In case it hasn't yet been included. -#include "multiply_constant.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -MultiplyConstantType::MultiplyConstantType( - const double scalar) : scalar(scalar) -{ - // Nothing to do here. -} - -template -void MultiplyConstantType::Forward( - const InputType& input, OutputType& output) -{ - output = input * scalar; -} - -template -void MultiplyConstantType::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) -{ - g = gy * scalar; -} - -template -template -void MultiplyConstantType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(scalar)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/multiply_merge_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/multiply_merge_impl.hpp deleted file mode 100644 index 596f61b445..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/multiply_merge_impl.hpp +++ /dev/null @@ -1,116 +0,0 @@ -/** - * @file methods/ann/layer/multiply_merge_impl.hpp - * @author Haritha Nair - * - * Definition of the MultiplyMerge module which multiplies the output of the - * given modules element-wise. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_MULTIPLY_MERGE_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_MULTIPLY_MERGE_IMPL_HPP - -// In case it hasn't yet been included. -#include "multiply_merge.hpp" - -// #include "../visitor/forward_visitor.hpp" -// #include "../visitor/backward_visitor.hpp" -// #include "../visitor/gradient_visitor.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -MultiplyMergeType::MultiplyMergeType( - const bool run) : - run(run) -{ - // Nothing to do here. -} - -// TODO: is this destructor needed? -template -MultiplyMergeType::~MultiplyMergeType() -{ - for (size_t i = 0; i < network.size(); ++i) - delete network[i]; -} - -template -void MultiplyMergeType::Forward( - const InputType& input, OutputType& output) -{ - InitializeForwardPassMemory(); - - if (run) - { - for (size_t i = 0; i < network.size(); ++i) - { - network[i]->Forward(input, layerOutputs[i]); - } - } - - output = layerOutputs.front(); - for (size_t i = 1; i < network.size(); ++i) - { - output %= layerOutputs[i]; - } -} - -template -void MultiplyMergeType::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) -{ - InitializeBackwardPassMemory(); - - if (run) - { - for (size_t i = 0; i < network.size(); ++i) - { - network[i]->Backward(layerOutputs[i], gy, layerDeltas[i]); - } - - g = layerDeltas.front(); - for (size_t i = 1; i < network.size(); ++i) - { - g += layerDeltas[i]; - } - } - else - { - g = gy; - } -} - -template -void MultiplyMergeType::Gradient( - const InputType& input, - const OutputType& error, - OutputType& /* gradient */ ) -{ - if (run) - { - for (size_t i = 0; i < network.size(); ++i) - { - network[i]->Gradient(input, error, layerGradients[i]); - } - } -} - -template -template -void MultiplyMergeType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(run)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/recurrent_attention_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/recurrent_attention_impl.hpp deleted file mode 100644 index 62d19bec87..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/recurrent_attention_impl.hpp +++ /dev/null @@ -1,237 +0,0 @@ -/** - * @file methods/ann/layer/recurrent_attention_impl.hpp - * @author Marcus Edel - * - * Implementation of the RecurrentAttention class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_RECURRENT_ATTENTION_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_RECURRENT_ATTENTION_IMPL_HPP - -// In case it hasn't yet been included. -#include "recurrent_attention.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -RecurrentAttention::RecurrentAttention() : - outSize(0), - rho(0), - forwardStep(0), - backwardStep(0), - deterministic(false) -{ - // Nothing to do. -} - -template -template -RecurrentAttention::RecurrentAttention( - const size_t outSize, - const RNNModuleType& rnn, - const ActionModuleType& action, - const size_t rho) : - outSize(outSize), - rnnModule(new RNNModuleType(rnn)), - actionModule(new ActionModuleType(action)), - rho(rho), - forwardStep(0), - backwardStep(0), - deterministic(false) -{ - network.push_back(rnnModule); - network.push_back(actionModule); -} - -template -void RecurrentAttention::Forward( - const InputType& input, OutputType& output) -{ - InitializeForwardPassMemory(); - - // Convenience naming. - OutputType& rnnOutput = layerOutputs.front(); - OutputType& actionOutput = layerOutputs.back(); - - // Initialize the action input. - if (initialInput.is_empty()) - { - initialInput = arma::zeros(outSize, input.n_cols); - } - - // Propagate through the action and recurrent module. - for (forwardStep = 0; forwardStep < rho; ++forwardStep) - { - if (forwardStep == 0) - { - actionModule->Forward(initialInput, actionOutput); - } - else - { - actionModule->Forward(rnnOutput, actionOutput); - } - - // Initialize the glimpse input. - InputType glimpseInput = arma::zeros(input.n_elem, 2); - glimpseInput.col(0) = input; - glimpseInput.submat(0, 1, actionOutput.n_elem - 1, 1) = - actionOutput; - - rnnModule->Forward(glimpseInput, rnnOutput); - - // Save the output parameter when training the module. - if (!deterministic) - { - for (size_t l = 0; l < network.size(); ++l) - { - // TODO: what if network[i] has a Model()? - // TODO: what does this actually do? do we need it? - moduleOutputParameter.push_back(network[l]->OutputParameter()); - } - } - } - - output = rnnOutput; - - forwardStep = 0; - backwardStep = 0; -} - -template -void RecurrentAttention::Backward( - const InputType& /* input */, - const OutputType& gy, - OutputType& g) -{ - InitializeBackwardPassMemory(); - - // Convenience names. - OutputType& rnnOutput = layerOutputs.front(); - OutputType& actionOutput = layerOutputs.back(); - OutputType& rnnGradient = layerGradients.front(); - OutputType& actionGradient = layerGradients.back(); - - if (intermediateGradient.is_empty() && backwardStep == 0) - { - // Initialize the attention gradients. - // TODO: do rnnModule or actionModule have a Model()? We may need to - // account for those weights too. - size_t weights = rnnModule->Parameters().n_elem + - actionModule->Parameters().n_elem; - - intermediateGradient = arma::zeros(weights, 1); - attentionGradient = arma::zeros(weights, 1); - - // Initialize the action error. - actionError = arma::zeros(actionOutput.n_rows, actionOutput.n_cols); - } - - // Propagate the attention gradients. - if (backwardStep == 0) - { - size_t offset = 0; - // TODO: what if rnnModule has a Model()? - rnnGradient = OutputType(intermediateGradient.memptr() + offset, - rnnModule->Parameters().n_rows, rnnModule->Parameters().n_cols, false, - false); - offset += rnnModule->Parameters().n_elem; - actionGradient = OutputType(intermediateGradient.memptr() + offset, - actionModule->Parameters().n_rows, actionModule->Parameters().n_cols, - false, false); - - attentionGradient.zeros(); - } - - // Back-propagate through time. - for (; backwardStep < rho; backwardStep++) - { - if (backwardStep == 0) - { - recurrentError = gy; - } - else - { - recurrentError = actionDelta; - } - - for (size_t l = 0; l < network.size(); ++l) - { - // TODO: handle case where HasModelCheck is true - network[network.size() - 1 - l] = moduleOutputParameter.back(); - moduleOutputParameter.pop_back(); - } - - if (backwardStep == (rho - 1)) - { - actionModule->Backward(actionOutput, actionError, actionDelta); - } - else - { - actionModule->Backward(initialInput, actionError, actionDelta); - } - - rnnModule->Backward(rnnOutput, recurrentError, rnnDelta); - - if (backwardStep == 0) - { - g = rnnDelta.col(1); - } - else - { - g += rnnDelta.col(1); - } - - IntermediateGradient(); - } -} - -template -void RecurrentAttention::Gradient( - const InputType& /* input */, - const OutputType& /* error */, - OutputType& /* gradient */) -{ - // Convenience naming. - OutputType& rnnGradient = layerGradients.front(); - OutputType& actionGradient = layerGradients.back(); - - size_t offset = 0; - // TODO: handle case where rnnModule or actionModule have a model - if (rnnModule->Parameters().n_elem != 0) - { - rnnGradient = attentionGradient.submat(offset, 0, offset + - rnnModule->Parameters().n_elem - 1, 0); - offset += rnnModule->Parameters().n_elem; - } - - if (actionModule->Parameters().n_elem != 0) - { - actionGradient = attentionGradient.submat(offset, 0, offset + - actionModule->Parameters().n_elem - 1, 0); - } -} - -template -template -void RecurrentAttention::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(rho)); - ar(CEREAL_NVP(outSize)); - ar(CEREAL_NVP(forwardStep)); - ar(CEREAL_NVP(backwardStep)); - - // TODO: lots of clearing? -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/recurrent_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/recurrent_impl.hpp deleted file mode 100644 index b99972c9b8..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/recurrent_impl.hpp +++ /dev/null @@ -1,284 +0,0 @@ -/** - * @file methods/ann/layer/recurrent_impl.hpp - * @author Marcus Edel - * - * Implementation of the Recurrent class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_RECURRENT_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_RECURRENT_IMPL_HPP - -// In case it hasn't yet been included. -#include "recurrent.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -Recurrent::Recurrent() : - rho(0), - forwardStep(0), - backwardStep(0), - gradientStep(0) -{ - // Nothing to do. -} - -template -template< - typename StartModuleType, - typename InputModuleType, - typename FeedbackModuleType, - typename TransferModuleType -> -Recurrent::Recurrent( - const StartModuleType& start, - const InputModuleType& input, - const FeedbackModuleType& feedback, - const TransferModuleType& transfer, - const size_t rho) : - startModule(new StartModuleType(start)), - inputModule(new InputModuleType(input)), - feedbackModule(new FeedbackModuleType(feedback)), - transferModule(new TransferModuleType(transfer)), - rho(rho), - forwardStep(0), - backwardStep(0), - gradientStep(0) -{ - initialModule = new SequentialType(); - mergeModule = new AddMerge(false, false, false); - recurrentModule = new SequentialType(false, false); - - initialModule->Add(inputModule); - initialModule->Add(startModule); - initialModule->Add(transferModule); - - mergeModule->Add(inputModule); - mergeModule->Add(feedbackModule); - - recurrentModule->Add(mergeModule); - recurrentModule->Add(transferModule); - - network.push_back(initialModule); - network.push_back(mergeModule); - network.push_back(feedbackModule); - network.push_back(recurrentModule); -} - -template -Recurrent::Recurrent( - const Recurrent& network) : - rho(network.rho), - forwardStep(network.forwardStep), - backwardStep(network.backwardStep), - gradientStep(network.gradientStep) -{ - startModule = network.startModule->Clone(); - inputModule = network.inputModule->Clone(); - feedbackModule = network.feedbackModule->Clone(); - transferModule = network.transferModule->Clone(); - - initialModule = new SequentialType(); - mergeModule = new AddMerge(false, false, false); - recurrentModule = new SequentialType(false, false); - - initialModule->Add(inputModule); - initialModule->Add(startModule); - initialModule->Add(transferModule); - - mergeModule->Add(inputModule); - mergeModule->Add(feedbackModule); - - recurrentModule->Add(mergeModule); - recurrentModule->Add(transferModule); - - this->network.push_back(initialModule); - this->network.push_back(mergeModule); - this->network.push_back(feedbackModule); - this->network.push_back(recurrentModule); -} - -template -void Recurrent::Forward( - const InputType& input, OutputType& output) -{ - InitializeForwardPassMemory(); - - // Convenience names. - OutputType& inputOutput = layerOutputs[0]; - OutputType& mergeOutput = layerOutputs[1]; - OutputType& feedbackOutput = layerOutputs[2]; - OutputType& recurrentOutput = layerOutputs[3]; - - if (forwardStep == 0) - { - initialModule->Forward(input, output); - } - else - { - inputModule->Forward(input, inputOutput); - // TODO: how to get transferModule output? - feedbackModule->Forward(transferModule->OutputParameter(), - feedbackOutput); - recurrentModule->Forward(input, output); - } - - // TODO: how to get transferModule output? - output = transferModule->OutputParameter(); - - // Save the feedback output parameter when training the module. - if (this->training) - { - feedbackOutputParameter.push_back(output); - } - - forwardStep++; - if (forwardStep == rho) - { - forwardStep = 0; - backwardStep = 0; - - if (!recurrentError.is_empty()) - { - recurrentError.zeros(); - } - } -} - -template -void Recurrent::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) -{ - InitializeBackwardPassMemory(); - - // Convenience names. - OutputType& inputOutput = layerOutputs[0]; - OutputType& mergeOutput = layerOutputs[1]; - OutputType& feedbackOutput = layerOutputs[2]; - OutputType& recurrentOutput = layerOutputs[3]; - OutputType& inputDelta = layerDeltas[0]; - OutputType& mergeDelta = layerDeltas[1]; - OutputType& feedbackDelta = layerDeltas[2]; - OutputType& recurrentDelta = layerDeltas[3]; - - if (!recurrentError.is_empty()) - { - recurrentError += gy; - } - else - { - recurrentError = gy; - } - - if (backwardStep < (rho - 1)) - { - recurrentModule->Backward(recurrentOutput, recurrentError, recurrentDelta); - inputModule->Backward(inputOutput, recurrentDelta, g); - feedbackModule->Backward(feedbackOutput, recurrentDelta, feedbackDelta); - } - else - { - // TODO: how to get these parameters? - initialModule->Backward(initialModule->OutputParameter(), recurrentError, - g); - } - - recurrentError = feedbackDelta; - backwardStep++; -} - -template -void Recurrent::Gradient( - const InputType& input, - const OutputType& error, - OutputType& /* gradient */) -{ - // Convenience names. - OutputType& inputOutput = layerOutputs[0]; - OutputType& mergeOutput = layerOutputs[1]; - OutputType& feedbackOutput = layerOutputs[2]; - OutputType& recurrentOutput = layerOutputs[3]; - OutputType& inputDelta = layerDeltas[0]; - OutputType& mergeDelta = layerDeltas[1]; - OutputType& feedbackDelta = layerDeltas[2]; - OutputType& recurrentDelta = layerDeltas[3]; - OutputType& inputGradient = layerGradients[0]; - OutputType& mergeGradient = layerGradients[1]; - OutputType& feedbackGradient = layerGradients[2]; - OutputType& recurrentGradient = layerGradients[3]; - - if (gradientStep < (rho - 1)) - { - recurrentModule->Gradient(input, error, recurrentGradient); - inputModule->Gradient(input, mergeDelta, inputGradient); - feedbackModule->Gradient( - feedbackOutputParameter[feedbackOutputParameter.size() - 2 - - gradientStep], mergeDelta, feedbackGradient); - } - else - { - recurrentGradient.zeros(); - inputGradient.zeros(); - feedbackGradient.zeros(); - - // TODO: how to do this? - initialModule->Gradient(input, startModule->Delta(), - initialModule->Gradient()); - } - - gradientStep++; - if (gradientStep == rho) - { - gradientStep = 0; - feedbackOutputParameter.clear(); - } -} - -template -template -void Recurrent::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - // TODO: overhaul this - - ar(CEREAL_POINTER(startModule)); - ar(CEREAL_POINTER(inputModule)); - ar(CEREAL_POINTER(feedbackModule)); - ar(CEREAL_POINTER(transferModule)); - ar(CEREAL_NVP(rho)); - - // Set up the network. - if (cereal::is_loading()) - { - initialModule = new SequentialType(); - mergeModule = new AddMerge(false, false, false); - recurrentModule = new SequentialType(false, false); - - initialModule->Add(inputModule); - initialModule->Add(startModule); - initialModule->Add(transferModule); - - mergeModule->Add(inputModule); - mergeModule->Add(feedbackModule); - - recurrentModule->Add(mergeModule); - recurrentModule->Add(transferModule); - - network.push_back(initialModule); - network.push_back(mergeModule); - network.push_back(feedbackModule); - network.push_back(recurrentModule); - } -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/reparametrization_impl.hpp deleted file mode 100644 index 72e9093242..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/reparametrization_impl.hpp +++ /dev/null @@ -1,152 +0,0 @@ -/** - * @file methods/ann/layer/reparametrization_impl.hpp - * @author Atharva Khandait - * - * Implementation of the Reparametrization layer class which samples from a - * gaussian distribution. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_IMPL_HPP - -// In case it hasn't yet been included. -#include "reparametrization.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -ReparametrizationType::ReparametrizationType( - const bool stochastic, - const bool includeKl, - const double beta) : - stochastic(stochastic), - includeKl(includeKl), - beta(beta) -{ - if (includeKl == false && beta != 1) - { - Log::Info << "The beta parameter will be ignored as KL divergence is not " - << "included." << std::endl; - } -} - -template -ReparametrizationType::ReparametrizationType( - const ReparametrizationType& layer) : - Layer(layer), - stochastic(layer.stochastic), - includeKl(layer.includeKl), - beta(layer.beta) -{ - // Nothing to do here. -} - -template -ReparametrizationType::ReparametrizationType( - ReparametrizationType&& layer) : - Layer(std::move(layer)), - stochastic(std::move(layer.stochastic)), - includeKl(std::move(layer.includeKl)), - beta(std::move(layer.beta)) -{ - // Nothing to do here. -} - -template -ReparametrizationType& -ReparametrizationType:: -operator=(const ReparametrizationType& layer) -{ - if (this != &layer) - { - Layer::operator=(layer); - stochastic = layer.stochastic; - includeKl = layer.includeKl; - beta = layer.beta; - } - - return *this; -} - -template -ReparametrizationType& -ReparametrizationType:: -operator=(ReparametrizationType&& layer) -{ - if (this != &layer) - { - Layer::operator=(std::move(layer)); - stochastic = std::move(layer.stochastic); - includeKl = std::move(layer.includeKl); - beta = std::move(layer.beta); - } - - return *this; -} - -template -void ReparametrizationType::Forward( - const InputType& input, OutputType& output) -{ - const size_t latentSize = this->outputDimensions[0]; - mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); - preStdDev = input.submat(0, 0, latentSize - 1, input.n_cols - 1); - - if (stochastic) - gaussianSample = arma::randn(latentSize, input.n_cols); - else - gaussianSample = arma::ones(latentSize, input.n_cols) * 0.7; - - SoftplusFunction::Fn(preStdDev, stdDev); - output = mean + stdDev % gaussianSample; -} - -template -void ReparametrizationType::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) -{ - OutputType tmp; - SoftplusFunction::Deriv(preStdDev, tmp); - - if (includeKl) - { - g = join_cols(gy % std::move(gaussianSample) % tmp + (-1 / stdDev + stdDev) - % tmp * beta, gy + mean * beta / mean.n_cols); - } - else - { - g = join_cols(gy % std::move(gaussianSample) % tmp, gy); - } -} - -template -double ReparametrizationType::Loss() -{ - if (!includeKl) - return 0; - - return -0.5 * beta * arma::accu(2 * arma::log(stdDev) - arma::pow(stdDev, 2) - - arma::pow(mean, 2) + 1) / mean.n_cols; -} - -template -template -void ReparametrizationType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(stochastic)); - ar(CEREAL_NVP(includeKl)); - ar(CEREAL_NVP(beta)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/sequential.hpp b/src/mlpack/methods/ann/layer/not_adapted/sequential.hpp deleted file mode 100644 index 471d5a1f35..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/sequential.hpp +++ /dev/null @@ -1,150 +0,0 @@ -/** - * @file methods/ann/layer/sequential.hpp - * @author Marcus Edel - * - * Definition of the Sequential class, which acts as a feed-forward fully - * connected network container. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_SEQUENTIAL_HPP -#define MLPACK_METHODS_ANN_LAYER_SEQUENTIAL_HPP - -#include - -#include "layer.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -/** - * Implementation of the Sequential class. The sequential class works as a - * feed-forward fully connected network container which plugs various layers - * together. - * - * This class can also be used as a container for a residual block. In that - * case, the sizes of the input and output matrices of this class should be - * equal. A typedef has been added for use as a Residual<> class. - * - * For more information, refer the following paper. - * - * @code - * @article{He15, - * author = {Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun}, - * title = {Deep Residual Learning for Image Recognition}, - * year = {2015}, - * url = {https://arxiv.org/abs/1512.03385}, - * eprint = {1512.03385}, - * } - * @endcode - * - * Note: If this class is used as the first layer of a network, it should be - * preceded by IdentityLayer<>. - * - * Note: This class should at least have two layers for a call to its Gradient() - * function. - * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - * @tparam Residual If true, use the object as a Residual block. - */ -template < - typename InputType = arma::mat, - typename OutputType = arma::mat, - bool Residual = false -> -class SequentialType : public MultiLayer -{ - public: - /** - * Create the Sequential object. - */ - SequentialType(); - - /** - * Create the Sequential object using the specified parameters. - * - * @param ownsLayers If true, then this module will delete its layers when - * deallocated. - */ - SequentialType(const bool ownsLayers); - - //! Copy constructor. - SequentialType(const SequentialType& layer); - - //! Copy assignment operator. - SequentialType& operator=(const SequentialType& layer); - - //! Destroy the Sequential object. - ~SequentialType(); - - //! Clone the SequentialType object. This handles polymorphism correctly. - SequentialType* Clone() const { return new SequentialType(*this); } - - /** - * Ordinary feed forward pass of a neural network, evaluating the function - * f(x) by propagating the activity forward through f. - * - * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. - */ - void Forward(const InputType& input, OutputType& output); - - /** - * Ordinary feed backward pass of a neural network, using 3rd-order tensors as - * input, calculating the function f(x) by propagating x backwards through f. - * Using the results from the feed forward pass. - * - * @param * (input) The propagated input activation. - * @param gy The backpropagated error. - * @param g The calculated gradient. - */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); - - /** - * Calculate the gradient using the output delta and the input activation. - * - * @param input The input parameter used for calculating the gradient. - * @param error The calculated error. - * @param gradient The calculated gradient. - */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& /* gradient */); - - size_t InputShape() const; - - /** - * Serialize the layer - */ - template - void serialize(Archive& ar, const uint32_t /* version */); - - private: - //! Indicator if we already initialized the model. - bool reset; - - //! Whether we are responsible for deleting the layers held in this module. - bool ownsLayers; -}; // class SequentialType - -// Standard Sequential layer. -typedef SequentialType Sequential; - -// Standard Residual layer. -typedef SequentialType Residual; - -} // namespace ann -} // namespace mlpack - -// Include implementation. -#include "sequential_impl.hpp" - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/sequential_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/sequential_impl.hpp deleted file mode 100644 index 6cac9d869f..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/sequential_impl.hpp +++ /dev/null @@ -1,163 +0,0 @@ -/** - * @file methods/ann/layer/sequential_impl.hpp - * @author Marcus Edel - * - * Implementation of the Sequential class, which acts as a feed-forward fully - * connected network container. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_SEQUENTIAL_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_SEQUENTIAL_IMPL_HPP - -// In case it hasn't yet been included. -#include "sequential.hpp" - -// TODO: can this be merged with MultiLayer more closely? - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -SequentialType:: -SequentialType() : - reset(false), ownsLayers(true) -{ - // Nothing to do here. -} - -template -SequentialType:: -SequentialType(const bool ownsLayers) : - reset(false), ownsLayers(ownsLayers) -{ - // Nothing to do here. -} - -template -SequentialType:: -SequentialType(const SequentialType& layer) : - reset(layer.reset), - ownsLayers(layer.ownsLayers) -{ - // Nothing to do here. -} - -template -SequentialType& -SequentialType:: -operator = (const SequentialType& layer) -{ - if (this != &layer) - { - reset = layer.reset; - ownsLayers = layer.ownsLayers; - network = layer.network; - // Call copy constructor of parent... - } - return *this; -} - - -template -SequentialType::~SequentialType() -{ - if (ownsLayers) - { - for (size_t i = 0; i < network.size(); ++i) - delete network[i]; - } -} - -template -void SequentialType:: -Forward(const InputType& input, OutputType& output) -{ - InitializeForwardPassMemory(); - - network.front()->Forward(input, layerOutputs.front()); - - for (size_t i = 1; i < network.size(); ++i) - { - network[i]->Forward(layerOutputs[i - 1], layerOutputs[i]); - } - - // TODO: optimization is possible here - output = layerOutputs.back(); - - if (Residual) - { - if (arma::size(output) != arma::size(input)) - { - Log::Fatal << "The sizes of the output and input matrices of the Residual" - << " block should be equal. Please examine the network architecture." - << std::endl; - } - output += input; - } -} - -template -void SequentialType::Backward( - const InputType& /* input */, - const OutputType& gy, - OutputType& g) -{ - InitializeBackwardPassMemory(); - - network.back()->Backward(layerOutputs.back(), gy, layerDeltas.back()); - - for (size_t i = 2; i < network.size() + 1; ++i) - { - network[network.size() - i]->Backward(layerOutputs[network.size() - i], - layerDeltas[network.size() - i + 1], layerDeltas[network.size() - i]); - } - - g = layerDeltas.front(); - - if (Residual) - { - g += gy; - } -} - -template -void SequentialType:: -Gradient(const InputType& input, - const OutputType& error, - OutputType& /* gradient */) -{ - InitializeGradientPassMemory(); - - network.back()->Gradient(layerOutputs[network.size() - 2], error, - layerGradients.back()); - - for (size_t i = 2; i < network.size(); ++i) - { - network[network.size() - i]->Gradient( - layerOutputs[network.size() - i - 1], - layerDeltas[network.size() - i + 1], - layerGradients[network.size() - i] - ); - } - - network.front()->Gradient(input, layerDeltas[1], layerGradients.front()); -} - -template -template -void SequentialType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(ownsLayers)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/weight_norm_impl.hpp b/src/mlpack/methods/ann/layer/not_adapted/weight_norm_impl.hpp deleted file mode 100644 index 16bfbd8ce5..0000000000 --- a/src/mlpack/methods/ann/layer/not_adapted/weight_norm_impl.hpp +++ /dev/null @@ -1,202 +0,0 @@ -/** - * @file methods/ann/layer/weight_norm_impl.hpp - * @author Toshal Agrawal - * - * Implementation of the WeightNorm Layer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ - -#ifndef MLPACK_METHODS_ANN_LAYER_WEIGHTNORM_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_WEIGHTNORM_IMPL_HPP - -// In case it is not included. -#include "weight_norm.hpp" - -namespace mlpack { -namespace ann { /** Artificial Neural Network. */ - -template -WeightNormType:: -WeightNormType() : wrappedLayer(new LinearType()) -{ - layerWeightSize = wrappedLayer->WeightSize(); - weights.set_size(layerWeightSize + 1, 1); - - layerWeights.set_size(layerWeightSize, 1); - layerGradients.set_size(layerWeightSize, 1); -} - -template -WeightNormType:: -WeightNormType(Layer* layer) : wrappedLayer(layer) -{ - layerWeightSize = wrappedLayer->WeightSize(); - weights.set_size(layerWeightSize + 1, 1); - - layerWeights.set_size(layerWeightSize, 1); - layerGradients.set_size(layerWeightSize, 1); -} - -template -WeightNormType::WeightNormType( - const WeightNormType& other) : - wrappedLayer(other.wrappedLayer->Clone()), - layerWeightSize(other.layerWeightSize), - weights(other.weights), - layerGradients(other.layerGradients), - layerWeights(other.layerWeights) -{ - // Nothing else to do. -} - -template -WeightNormType::WeightNormType( - WeightNormType&& other) : - wrappedLayer(std::move(other.wrappedLayer)), - layerWeightSize(other.layerWeightSize), - weights(std::move(other.weights)), - layerGradients(std::move(other.layerGradients)), - layerWeights(std::move(other.layerWeights)) -{ - // Reset the other layer. - other = WeightNormType(); -} - -template -WeightNormType& -WeightNormType::operator=( - const WeightNormType& other) -{ - if (this != &other) - { - wrappedLayer = other.wrappedLayer->Clone(); - layerWeightSize = other.layerWeightSize; - weights = other.weights; - layerWeights = other.layerWeights; - layerGradients = other.layerGradients; - } - - return *this; -} - -template -WeightNormType& -WeightNormType::operator=( - WeightNormType&& other) -{ - if (this != &other) - { - wrappedLayer = std::move(other.wrappedLayer); - layerWeightSize = other.layerWeightSize; - weights = std::move(other.weights); - layerWeights = std::move(other.layerWeights); - layerGradients = std::move(other.layerGradients); - - // Reset the other layer. - other = WeightNormType(); - } - - return *this; -} - -template -WeightNormType::~WeightNormType() -{ - delete wrappedLayer; -} - -template -void WeightNormType::SetWeights( - typename OutputType::elem_type* weightsPtr) -{ - // Set the weights of the inside layer to layerWeights. - // This is done to set the non-bias terms correctly. - /* boost::apply_visitor(WeightSetVisitor(layerWeights, 0), wrappedLayer); */ - wrappedLayer->SetWeights(weightsPtr); - wrappedLayer->Parameters() = OutputType(layerWeights.memptr(), - wrappedLayer->Parameters().n_rows, wrappedLayer->Parameters().n_cols, - false, false); - - /* biasWeightSize = boost::apply_visitor(BiasSetVisitor(weights, 0), */ - /* wrappedLayer); */ - biasWeightSize = 0; - - vectorParameter = OutputType(weights.memptr() + biasWeightSize, - layerWeightSize - biasWeightSize, 1, false, false); - - scalarParameter = OutputType(weights.memptr() + layerWeightSize, 1, 1, false, - false); -} - -template -void WeightNormType::Forward( - const InputType& input, OutputType& output) -{ - // Initialize the non-bias weights of wrapped layer. - const double normVectorParameter = arma::norm(vectorParameter, 2); - layerWeights.rows(0, layerWeightSize - biasWeightSize - 1) = - scalarParameter(0) * vectorParameter / normVectorParameter; - - wrappedLayer->Forward(input, output); -} - -template -void WeightNormType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) -{ - wrappedLayer->Backward(input, gy, g); -} - -// TODO: this part is not trivial... -template -void WeightNormType::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) -{ - ResetGradients(layerGradients); - - // Calculate the gradients of the wrapped layer. - wrappedLayer->Gradient(input, error, gradient); - - // Store the norm of vector parameter temporarily. - const double normVectorParameter = arma::norm(vectorParameter, 2); - - // Set the gradients of the bias terms. - if (biasWeightSize != 0) - { - gradient.rows(0, biasWeightSize - 1) = OutputType(layerGradients.memptr() + - layerWeightSize - biasWeightSize, biasWeightSize, 1, false, false); - } - - // Calculate the gradients of the scalar parameter. - gradient[gradient.n_rows - 1] = arma::accu(layerGradients.rows(0, - layerWeightSize - biasWeightSize - 1) % vectorParameter) / - normVectorParameter; - - // Calculate the gradients of the vector parameter. - gradient.rows(biasWeightSize, layerWeightSize - 1) = - scalarParameter(0) / normVectorParameter * (layerGradients.rows(0, - layerWeightSize - biasWeightSize - 1) - gradient[gradient.n_rows - 1] / - normVectorParameter * vectorParameter); -} - -template -template -void WeightNormType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class*>(this)); - - ar(CEREAL_POINTER(wrappedLayer)); - ar(CEREAL_NVP(layerWeightSize)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/padding.hpp b/src/mlpack/methods/ann/layer/padding.hpp index 018a28479b..b7bfcab976 100644 --- a/src/mlpack/methods/ann/layer/padding.hpp +++ b/src/mlpack/methods/ann/layer/padding.hpp @@ -13,20 +13,25 @@ #define MLPACK_METHODS_ANN_LAYER_PADDING_HPP #include -#include "layer.hpp" +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Implementation of the Padding module class. The Padding module applies - * (zero-valued) padding on the input data. + * Implementation of the Padding module class. The Padding module applies a bias term + * to the incoming data. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class PaddingType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class Padding { public: /** @@ -39,13 +44,12 @@ class PaddingType : public Layer * @param inputWidth Width of the input. * @param inputHeight Height of the input. */ - PaddingType(const size_t padWLeft = 0, - const size_t padWRight = 0, - const size_t padHTop = 0, - const size_t padHBottom = 0); - - //! Clone the PaddingType object. This handles polymorphism correctly. - PaddingType* Clone() const { return new PaddingType(*this); } + Padding(const size_t padWLeft = 0, + const size_t padWRight = 0, + const size_t padHTop = 0, + const size_t padHBottom = 0, + const size_t inputWidth = 0, + const size_t inputHeight = 0); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -54,7 +58,8 @@ class PaddingType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -65,9 +70,20 @@ class PaddingType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const MatType& /* input */, - const MatType& gy, - MatType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the left padding width. size_t PadWLeft() const { return padWLeft; } @@ -89,8 +105,25 @@ class PaddingType : public Layer //! Modify the bottom padding width. size_t& PadHBottom() { return padHBottom; } - //! Compute the output dimensions of the layer using `InputDimensions()`. - void ComputeOutputDimensions(); + //! Get the input width. + size_t InputWidth() const { return inputWidth; } + //! Modify the input width. + size_t& InputWidth() { return inputWidth; } + + //! Get the input height. + size_t InputHeight() const { return inputHeight; } + //! Modify the input height. + size_t& InputHeight() { return inputHeight; } + + //! Get the output width. + size_t OutputWidth() const { return outputWidth; } + //! Modify the output width. + size_t& OutputWidth() { return outputWidth; } + + //! Get the output height. + size_t OutputHeight() const { return outputHeight; } + //! Modify the output height. + size_t& OutputHeight() { return outputHeight; } /** * Serialize the layer. @@ -111,12 +144,36 @@ class PaddingType : public Layer //! Locally-stored bottom padding height. size_t padHBottom; - //! Cached number of input maps. - size_t totalInMaps; -}; // class PaddingType + //! Locally-stored number of rows and columns of input. + size_t nRows, nCols; -// Standard Padding layer. -typedef PaddingType Padding; + //! Locally-stored input height. + size_t inputHeight; + + //! Locally-stored input width. + size_t inputWidth; + + //! Locally-stored output height. + size_t outputHeight; + + //! Locally-stored output width. + size_t outputWidth; + + //! Locally-stored number of input channels. + size_t inSize; + + //! Locally-stored cube input parameter. + arma::cube inputTemp; + + //! Locally-stored output parameter. + arma::cube outputTemp; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class Padding } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/padding_impl.hpp b/src/mlpack/methods/ann/layer/padding_impl.hpp index 2c2811209b..9a57f69458 100644 --- a/src/mlpack/methods/ann/layer/padding_impl.hpp +++ b/src/mlpack/methods/ann/layer/padding_impl.hpp @@ -19,121 +19,85 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -PaddingType::PaddingType( +template +Padding::Padding( const size_t padWLeft, const size_t padWRight, const size_t padHTop, - const size_t padHBottom) : + const size_t padHBottom, + const size_t inputWidth, + const size_t inputHeight) : padWLeft(padWLeft), padWRight(padWRight), padHTop(padHTop), padHBottom(padHBottom), - totalInMaps(0) + nRows(0), + nCols(0), + inputHeight(inputWidth), + inputWidth(inputHeight), + inSize(0) { // Nothing to do here. } -template -void PaddingType::Forward(const MatType& input, MatType& output) +template +template +void Padding::Forward( + const arma::Mat& input, arma::Mat& output) { - // Make an alias of the input and output so that we can deal with the first - // two dimensions directly. - arma::Cube reshapedInput( - (typename MatType::elem_type*) input.memptr(), - this->inputDimensions[0], this->inputDimensions[1], totalInMaps * - input.n_cols, false, true); - arma::Cube reshapedOutput(output.memptr(), - this->outputDimensions[0], this->outputDimensions[1], totalInMaps * - output.n_cols, false, true); + nRows = input.n_rows; + nCols = input.n_cols; - // Set the padding parts to 0. - if (padWLeft > 0) + if (inputWidth == 0 || inputHeight == 0) { - reshapedOutput.tube(0, - 0, - reshapedOutput.n_rows - 1, - padWLeft - 1).zeros(); + output = arma::zeros(nRows + padWLeft + padWRight, + nCols + padHTop + padHBottom); + output.submat(padWLeft, padHTop, padWLeft + nRows - 1, + padHTop + nCols - 1) = input; + } + else + { + inSize = input.n_elem / (inputWidth * inputHeight * nCols); + inputTemp = arma::Cube(const_cast&>(input).memptr(), + inputWidth, inputHeight, inSize * nCols, false, false); + outputTemp = arma::zeros>(inputWidth + padWLeft + padWRight, + inputHeight + padHTop + padHBottom, inSize * nCols); + for (size_t i = 0; i < inputTemp.n_slices; ++i) + { + outputTemp.slice(i).submat(padWLeft, padHTop, padWLeft + inputWidth - 1, + padHTop + inputHeight - 1) = inputTemp.slice(i); + } + + output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem / nCols, + nCols); } - if (padHTop > 0) - { - reshapedOutput.tube(0, - padWLeft, - padHTop - 1, - padWLeft + this->inputDimensions[1] - 1).zeros(); - } - - if (padWRight > 0) - { - reshapedOutput.tube(0, - padWLeft + this->inputDimensions[1], - reshapedOutput.n_rows - 1, - reshapedOutput.n_cols - 1).zeros(); - } - - if (padHBottom > 0) - { - reshapedOutput.tube(padHTop + this->inputDimensions[0], - padWLeft, - reshapedOutput.n_rows - 1, - padWLeft + this->inputDimensions[1] - 1).zeros(); - } - - // Copy the input matrix. - reshapedOutput.tube(padHTop, - padWLeft, - padHTop + this->inputDimensions[0] - 1, - padWLeft + this->inputDimensions[1] - 1) = reshapedInput; + outputWidth = inputWidth + padWLeft + padWRight; + outputHeight = inputHeight + padHTop + padHBottom; } -template -void PaddingType::Backward( - const MatType& /* input */, - const MatType& gy, - MatType& g) +template +template +void Padding::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - // Reshape g and gy so that extracting the un-padded input is easier to - // understand. - arma::Cube reshapedGy( - (typename MatType::elem_type*) gy.memptr(), this->outputDimensions[0], - this->outputDimensions[1], totalInMaps * gy.n_cols, false, true); - arma::Cube reshapedG(g.memptr(), - this->inputDimensions[0], this->inputDimensions[1], totalInMaps * - g.n_cols, false, true); - - reshapedG = reshapedGy.tube(padHTop, - padWLeft, - padHTop + this->inputDimensions[0] - 1, - padWLeft + this->inputDimensions[1] - 1); + g = gy.submat(padWLeft, padHTop, padWLeft + nRows - 1, + padHTop + nCols - 1); } -template -void PaddingType::ComputeOutputDimensions() -{ - this->outputDimensions = this->inputDimensions; - - this->outputDimensions[0] += padHTop + padHBottom; - this->outputDimensions[1] += padWLeft + padWRight; - - // Higher dimensions remain unchanged. But, we will cache the product of - // these higher dimensions. - totalInMaps = 1; - for (size_t i = 2; i < this->inputDimensions.size(); ++i) - totalInMaps *= this->inputDimensions[i]; -} - -template +template template -void PaddingType::serialize(Archive& ar, const uint32_t /* version */) +void Padding::serialize( + Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(padWLeft)); ar(CEREAL_NVP(padWRight)); ar(CEREAL_NVP(padHTop)); ar(CEREAL_NVP(padHBottom)); - ar(CEREAL_NVP(totalInMaps)); + ar(CEREAL_NVP(inputWidth)); + ar(CEREAL_NVP(inputHeight)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/parametric_relu.hpp b/src/mlpack/methods/ann/layer/parametric_relu.hpp similarity index 64% rename from src/mlpack/methods/ann/layer/not_adapted/parametric_relu.hpp rename to src/mlpack/methods/ann/layer/parametric_relu.hpp index 5fc3ff83a3..f40be33b2a 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/parametric_relu.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu.hpp @@ -17,8 +17,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -35,14 +33,16 @@ namespace ann /** Artificial Neural Network. */ { * \right. * @f} * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class PReLUType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class PReLU { public: /** @@ -53,13 +53,12 @@ class PReLUType : public Layer * * @param userAlpha Non zero gradient */ - PReLUType(const double userAlpha = 0.03); + PReLU(const double userAlpha = 0.03); - //! Clone the PReLUType object. This handles polymorphism correctly. - PReLUType* Clone() const { return new PReLUType(*this); } - - //! Reset the layer parameter. - void SetWeights(typename OutputType::elem_type* weightsPtr); + /* + * Reset the layer parameter. + */ + void Reset(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -68,6 +67,7 @@ class PReLUType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -79,7 +79,8 @@ class PReLUType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const DataType& input, const DataType& gy, DataType& g); /** * Calculate the gradient using the output delta and the input activation. @@ -88,14 +89,30 @@ class PReLUType : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - OutputType const& Parameters() const { return alpha; } + OutputDataType const& Parameters() const { return alpha; } //! Modify the parameters. - OutputType& Parameters() { return alpha; } + OutputDataType& Parameters() { return alpha; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the non zero gradient. double const& Alpha() const { return alpha(0); } @@ -112,18 +129,22 @@ class PReLUType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Leakyness Parameter object. - OutputType alpha; + OutputDataType alpha; + + //! Locally-stored gradient object. + OutputDataType gradient; //! Leakyness Parameter given by user in the range 0 < alpha < 1. double userAlpha; }; // class PReLU -// Convenience typedefs. - -// Standard PReLU layer. -typedef PReLUType PReLU; - } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp similarity index 51% rename from src/mlpack/methods/ann/layer/not_adapted/parametric_relu_impl.hpp rename to src/mlpack/methods/ann/layer/parametric_relu_impl.hpp index 97ce441c09..a584603f5e 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/parametric_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp @@ -12,8 +12,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_ANN_LAYER_PRELU_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_PRELU_IMPL_HPP +#ifndef MLPACK_METHODS_ANN_LAYER_PReLU_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_PReLU_IMPL_HPP // In case it hasn't yet been included. #include "parametric_relu.hpp" @@ -21,66 +21,66 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -PReLUType::PReLUType( +template +PReLU::PReLU( const double userAlpha) : userAlpha(userAlpha) { alpha.set_size(WeightSize(), 1); alpha(0) = userAlpha; } -template -void PReLUType::SetWeights( - typename OutputType::elem_type* weightsPtr) +template +void PReLU::Reset() { - alpha = arma::mat(weightsPtr, 1, 1, false, false); - //! Set value of alpha to the one given by user. - // TODO: this doesn't even make any sense. is it trainable or not? - // why is there userAlpha? is that for initialization only? - alpha(0) = userAlpha; + alpha = arma::mat(alpha.memptr(), 1, 1, false, false); } +template template -void PReLUType::Forward( +void PReLU::Forward( const InputType& input, OutputType& output) { - // TODO: use transform()? - output = input; - arma::uvec negative = arma::find(input < 0); - output(negative) = input(negative) * alpha(0); + output = arma::max(input, alpha(0) * input); } -template -void PReLUType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) +template +template +void PReLU::Backward( + const DataType& input, const DataType& gy, DataType& g) { - OutputType derivative; + DataType derivative; derivative.set_size(arma::size(input)); for (size_t i = 0; i < input.n_elem; ++i) + { derivative(i) = (input(i) >= 0) ? 1 : alpha(0); + } g = gy % derivative; } -template -void PReLUType::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) +template +template +void PReLU::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - OutputType zeros = arma::zeros(input.n_rows, input.n_cols); + if (gradient.n_elem == 0) + { + gradient = arma::zeros(1, 1); + } + + arma::mat zeros = arma::zeros(input.n_rows, input.n_cols); gradient(0) = arma::accu(error % arma::min(zeros, input)) / input.n_cols; } -template +template template -void PReLUType::serialize( +void PReLU::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(alpha)); } diff --git a/src/mlpack/methods/ann/layer/not_adapted/pixel_shuffle.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/pixel_shuffle.hpp rename to src/mlpack/methods/ann/layer/pixel_shuffle.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/pixel_shuffle_impl.hpp b/src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/pixel_shuffle_impl.hpp rename to src/mlpack/methods/ann/layer/pixel_shuffle_impl.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/positional_encoding.hpp b/src/mlpack/methods/ann/layer/positional_encoding.hpp similarity index 63% rename from src/mlpack/methods/ann/layer/not_adapted/positional_encoding.hpp rename to src/mlpack/methods/ann/layer/positional_encoding.hpp index a1317b3e52..8678426414 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/positional_encoding.hpp +++ b/src/mlpack/methods/ann/layer/positional_encoding.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/positional_encoding.hpp * @author Mrityunjay Tripathi @@ -26,22 +25,22 @@ namespace ann /** Artificial Neural Network. */ { * `(embedDim * maxSequenceLength, batchSize)`. The embeddings are stored * consequently. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class PositionalEncodingType : public Layer +class PositionalEncoding { public: /** - * Create PositionalEncodingType object. + * Create PositionalEncoding object. */ - PositionalEncodingType(); + PositionalEncoding(); /** * Create the PositionalEncoding layer object using the specified parameters. @@ -49,11 +48,8 @@ class PositionalEncodingType : public Layer * @param embedDim The length of the embedding vector. * @param maxSequenceLength Number of tokens in each sequence. */ - PositionalEncodingType(const size_t embedDim, - const size_t maxSequenceLength); - - //! Clone the PositionalEncodingType object. This handles polymorphism correctly. - PositionalEncodingType* Clone() const { return new PositionalEncodingType(*this); } + PositionalEncoding(const size_t embedDim, + const size_t maxSequenceLength); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -62,7 +58,8 @@ class PositionalEncodingType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -73,12 +70,28 @@ class PositionalEncodingType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the positional encoding vector. - InputType const& Encoding() const { return positionalEncoding; } + InputDataType const& Encoding() const { return positionalEncoding; } size_t InputShape() const { @@ -104,11 +117,17 @@ class PositionalEncodingType : public Layer size_t maxSequenceLength; //! Locally-stored positional encodings. - InputType positionalEncoding; -}; // class PositionalEncodingTest + InputDataType positionalEncoding; -// Standard PositionalEncoding layer. -typedef PositionalEncodingType PositionalEncoding; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class PositionalEncoding } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/positional_encoding_impl.hpp b/src/mlpack/methods/ann/layer/positional_encoding_impl.hpp similarity index 59% rename from src/mlpack/methods/ann/layer/not_adapted/positional_encoding_impl.hpp rename to src/mlpack/methods/ann/layer/positional_encoding_impl.hpp index 30ad88ae2f..8dc63a5894 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/positional_encoding_impl.hpp +++ b/src/mlpack/methods/ann/layer/positional_encoding_impl.hpp @@ -19,16 +19,16 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -PositionalEncodingType::PositionalEncodingType() : +template +PositionalEncoding::PositionalEncoding() : embedDim(0), maxSequenceLength(0) { // Nothing to do here. } -template -PositionalEncodingType::PositionalEncodingType( +template +PositionalEncoding::PositionalEncoding( const size_t embedDim, const size_t maxSequenceLength) : embedDim(embedDim), @@ -37,14 +37,14 @@ PositionalEncodingType::PositionalEncodingType( InitPositionalEncoding(); } -template -void PositionalEncodingType::InitPositionalEncoding() +template +void PositionalEncoding::InitPositionalEncoding() { positionalEncoding.set_size(maxSequenceLength, embedDim); - const InputType position = arma::regspace(0, 1, maxSequenceLength - 1); - const InputType divTerm = arma::exp(arma::regspace(0, 2, embedDim - 1) + const InputDataType position = arma::regspace(0, 1, maxSequenceLength - 1); + const InputDataType divTerm = arma::exp(arma::regspace(0, 2, embedDim - 1) * (- std::log(10000.0) / embedDim)); - const InputType theta = position * divTerm.t(); + const InputDataType theta = position * divTerm.t(); for (size_t i = 0; i < theta.n_cols; ++i) { positionalEncoding.col(2 * i) = arma::sin(theta.col(i)); @@ -53,9 +53,10 @@ void PositionalEncodingType::InitPositionalEncoding() positionalEncoding = arma::vectorise(positionalEncoding.t()); } -template -void PositionalEncodingType::Forward( - const InputType& input, OutputType& output) +template +template +void PositionalEncoding::Forward( + const arma::Mat& input, arma::Mat& output) { if (input.n_rows != embedDim * maxSequenceLength) Log::Fatal << "Incorrect input dimensions!" << std::endl; @@ -63,20 +64,19 @@ void PositionalEncodingType::Forward( output = input.each_col() + positionalEncoding; } -template -void PositionalEncodingType::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) +template +template +void PositionalEncoding::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { g = gy; } -template +template template -void PositionalEncodingType::serialize( +void PositionalEncoding::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(embedDim)); ar(CEREAL_NVP(maxSequenceLength)); diff --git a/src/mlpack/methods/ann/layer/radial_basis_function.hpp b/src/mlpack/methods/ann/layer/radial_basis_function.hpp index 9865ede4d4..6f84303360 100644 --- a/src/mlpack/methods/ann/layer/radial_basis_function.hpp +++ b/src/mlpack/methods/ann/layer/radial_basis_function.hpp @@ -10,21 +10,22 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_ANN_LAYER_RADIAL_BASIS_FUNCTION_HPP -#define MLPACK_METHODS_ANN_LAYER_RADIAL_BASIS_FUNCTION_HPP +#ifndef MLPACK_METHODS_ANN_LAYER_RBF_HPP +#define MLPACK_METHODS_ANN_LAYER_RBF_HPP #include #include -#include "layer.hpp" +#include "layer_types.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { + /** - * Implementation of the Radial Basis Function layer. The RBFType class, when - * used with a non-linear activation function, acts as a Radial Basis Function - * which can be used with a feed-forward neural network. + * Implementation of the Radial Basis Function layer. The RBF class when use with a + * non-linear activation function acts as a Radial Basis Function which can be used + * with Feed-Forward neural network. * * For more information, refer to the following paper, * @@ -37,47 +38,37 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). * @tparam Activation Type of the activation function (mlpack::ann::Gaussian). */ template < - typename MatType = arma::mat, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, typename Activation = GaussianFunction > -class RBFType : public Layer +class RBF { public: - //! Create the RBFType object. - RBFType(); + //! Create the RBF object. + RBF(); /** * Create the Radial Basis Function layer object using the specified * parameters. * + * @param inSize The number of input units. * @param outSize The number of output units. * @param centres The centres calculated using k-means of data. * @param betas The beta value to be used with centres. */ - RBFType(const size_t outSize, - MatType& centres, - double betas = 0); - - //! Clone the LinearType object. This handles polymorphism correctly. - RBFType* Clone() const { return new RBFType(*this); } - - // Virtual destructor. - virtual ~RBFType() { } - - //! Copy the given RBFType layer. - RBFType(const RBFType& other); - //! Take ownership of the given RBFType layer. - RBFType(RBFType&& other); - //! Copy the given RBFType layer. - RBFType& operator=(const RBFType& other); - //! Take ownership of the given RBFType layer. - RBFType& operator=(RBFType&& other); + RBF(const size_t inSize, + const size_t outSize, + arma::mat& centres, + double betas = 0); /** * Ordinary feed forward pass of the radial basis function. @@ -85,21 +76,51 @@ class RBFType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const MatType& input, MatType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the radial basis function. + * */ - void Backward(const MatType& /* input */, - const MatType& /* gy */, - MatType& /* g */); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& /* gy */, + arma::Mat& /* g */); - //! Compute the output dimensions of the layer given `InputDimensions()`. The - //! RBFType layer flattens the input. - void ComputeOutputDimensions(); + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + //! Get the parameters. + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the input size. + size_t InputSize() const { return inSize; } + + //! Get the output size. + size_t OutputSize() const { return outSize; } + + //! Get the detla. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the size of the weights. - size_t WeightSize() const { return 0; } + size_t WeightSize() const + { + return 0; + } + + //! Get the shape of the input. + size_t InputShape() const + { + return inSize; + } /** * Serialize the layer. @@ -108,20 +129,33 @@ class RBFType : public Layer void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored number of input units. + size_t inSize; + //! Locally-stored number of output units. size_t outSize; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored the sigmas values. + double sigmas; + //! Locally-stored the betas values. double betas; //! Locally-stored the learnable centre of the shape. - MatType centres; + InputDataType centres; + + //! Locally-stored input parameter object. + InputDataType inputParameter; //! Locally-stored the output distances of the shape. - MatType distances; -}; // class RBFType - -typedef RBFType RBF; + OutputDataType distances; +}; // class RBF } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp b/src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp index 55887792cd..8ba09b0ab6 100644 --- a/src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp +++ b/src/mlpack/methods/ann/layer/radial_basis_function_impl.hpp @@ -2,13 +2,14 @@ * @file radial_basis_function_impl.hpp * @author Himanshu Pathak * + * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_ANN_LAYER_RADIAL_BASIS_FUNCTION_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_RADIAL_BASIS_FUNCTION_IMPL_HPP +#ifndef MLPACK_METHODS_ANN_LAYER_RBF_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_RBF_IMPL_HPP // In case it hasn't yet been included. #include "radial_basis_function.hpp" @@ -16,146 +17,85 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -RBFType::RBFType() : - Layer(), +template +RBF::RBF() : + inSize(0), outSize(0), + sigmas(0), betas(0) { // Nothing to do here. } -template -RBFType::RBFType( +template +RBF::RBF( + const size_t inSize, const size_t outSize, - MatType& centres, + arma::mat& centres, double betas) : + inSize(inSize), outSize(outSize), betas(betas), centres(centres) { - double sigmas = 0; + sigmas = 0; if (betas == 0) { for (size_t i = 0; i < centres.n_cols; i++) { - double maxDis = 0; - MatType temp = centres.each_col() - centres.col(i); - maxDis = arma::accu(arma::max(arma::pow(arma::sum( + double max_dis = 0; + arma::mat temp = centres.each_col() - centres.col(i); + max_dis = arma::accu(arma::max(arma::pow(arma::sum( arma::pow((temp), 2), 0), 0.5).t())); - if (maxDis > sigmas) - sigmas = maxDis; + if (max_dis > sigmas) + sigmas = max_dis; } this->betas = std::pow(2 * outSize, 0.5) / sigmas; } } -template -RBFType::RBFType(const RBFType& other) : - Layer(other), - outSize(other.outSize), - betas(other.betas), - centres(other.centres) +template +void RBF::Forward( + const arma::Mat& input, + arma::Mat& output) { - // Nothing to do. -} - -template -RBFType::RBFType(RBFType&& other) : - Layer(other), - outSize(other.outSize), - betas(other.betas), - centres(std::move(other.centres)) -{ - // Nothing to do. -} - -template -RBFType& -RBFType::operator=(const RBFType& other) -{ - if (&other != this) - { - Layer::operator=(other); - outSize = other.outSize; - betas = other.betas; - centres = other.centres; - } - - return *this; -} - -template -RBFType& -RBFType::operator=(RBFType&& other) -{ - if (&other != this) - { - Layer::operator=(std::move(other)); - outSize = std::move(other.outSize); - betas = std::move(other.betas); - centres = std::move(other.centres); - } - - return *this; -} - -template -void RBFType::Forward( - const MatType& input, - MatType& output) -{ - // Sanity check: make sure the dimensions are right. - if (input.n_rows != centres.n_rows) - { - Log::Fatal << "RBFType::Forward(): input size (" << input.n_rows << ") does" - << " not match given center size (" << centres.n_rows << ")!" - << std::endl; - } - - distances = MatType(outSize, input.n_cols); + distances = arma::mat(outSize, input.n_cols); for (size_t i = 0; i < input.n_cols; i++) { - MatType temp = centres.each_col() - input.col(i); + arma::mat temp = centres.each_col() - input.col(i); distances.col(i) = arma::pow(arma::sum( - arma::pow((temp), 2), 0), 0.5).t(); + arma::pow((temp), 2), 0), 0.5).t(); } - Activation::Fn(distances * std::pow(betas, 0.5), output); + Activation::Fn(distances * std::pow(betas, 0.5), + output); } -template -void RBFType::Backward( - const MatType& /* input */, - const MatType& /* gy */, - MatType& /* g */) +template +template +void RBF::Backward( + const arma::Mat& /* input */, + const arma::Mat& /* gy */, + arma::Mat& /* g */) { // Nothing to do here. } -template -void RBFType::ComputeOutputDimensions() -{ - this->outputDimensions = std::vector(this->inputDimensions.size(), 1); - - // This flattens the input. - this->outputDimensions[0] = outSize; -} - -template +template template -void RBFType::serialize( +void RBF::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(distances)); ar(CEREAL_NVP(centres)); - ar(CEREAL_NVP(betas)); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/recurrent.hpp b/src/mlpack/methods/ann/layer/recurrent.hpp similarity index 54% rename from src/mlpack/methods/ann/layer/not_adapted/recurrent.hpp rename to src/mlpack/methods/ann/layer/recurrent.hpp index 836d1aa4b8..b82fcb175b 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/recurrent.hpp +++ b/src/mlpack/methods/ann/layer/recurrent.hpp @@ -14,6 +14,12 @@ #include +#include "../visitor/delete_visitor.hpp" +#include "../visitor/delta_visitor.hpp" +#include "../visitor/copy_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" +#include "../visitor/input_shape_visitor.hpp" + #include "layer_types.hpp" #include "add_merge.hpp" #include "sequential.hpp" @@ -25,16 +31,17 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the RecurrentLayer class. Recurrent layers can be used * similarly to feed-forward layers. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename... CustomLayers > -class Recurrent : public MultiLayer +class Recurrent { public: /** @@ -72,7 +79,8 @@ class Recurrent : public MultiLayer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -83,9 +91,10 @@ class Recurrent : public MultiLayer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -94,9 +103,38 @@ class Recurrent : public MultiLayer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& /* gradient */); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */); + + //! Get the model modules. + std::vector >& Model() { return network; } + + //! The value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } + + //! Get the parameters. + OutputDataType const& Parameters() const { return parameters; } + //! Modify the parameters. + OutputDataType& Parameters() { return parameters; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the number of steps to backpropagate through time. size_t const& Rho() const { return rho; } @@ -111,17 +149,23 @@ class Recurrent : public MultiLayer void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delete visitor module object. + DeleteVisitor deleteVisitor; + + //! Locally-stored copy visitor + CopyVisitor copyVisitor; + //! Locally-stored start module. - Layer* startModule; + LayerTypes startModule; //! Locally-stored input module. - Layer* inputModule; + LayerTypes inputModule; //! Locally-stored feedback module. - Layer* feedbackModule; + LayerTypes feedbackModule; //! Locally-stored transfer module. - Layer* transferModule; + LayerTypes transferModule; //! Number of steps to backpropagate through time (BPTT). size_t rho; @@ -135,26 +179,48 @@ class Recurrent : public MultiLayer //! Locally-stored number of gradient steps. size_t gradientStep; + //! If true dropout and scaling is disabled, see notes above. + bool deterministic; + + //! To know whether this object allocated memory. We need this to know + //! whether we should delete the metric member variable in the destructor. + bool ownsLayer; + //! Locally-stored weight object. - OutputType parameters; + OutputDataType parameters; //! Locally-stored initial module. - SequentialType* initialModule; + LayerTypes initialModule; //! Locally-stored recurrent module. - SequentialType* recurrentModule; + LayerTypes recurrentModule; //! Locally-stored model modules. - std::vector*> network; + std::vector > network; //! Locally-stored merge module. - AddMerge* mergeModule; + LayerTypes mergeModule; + + //! Locally-stored delta visitor. + DeltaVisitor deltaVisitor; + + //! Locally-stored output parameter visitor. + OutputParameterVisitor outputParameterVisitor; //! Locally-stored feedback output parameters. - std::vector feedbackOutputParameter; + std::vector feedbackOutputParameter; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; //! Locally-stored recurrent error parameter. - OutputType recurrentError; + arma::mat recurrentError; }; // class Recurrent } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/recurrent_attention.hpp b/src/mlpack/methods/ann/layer/recurrent_attention.hpp similarity index 57% rename from src/mlpack/methods/ann/layer/not_adapted/recurrent_attention.hpp rename to src/mlpack/methods/ann/layer/recurrent_attention.hpp index 7783a3f4c9..63838dc479 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/recurrent_attention.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_attention.hpp @@ -14,6 +14,11 @@ #include +#include "../visitor/delta_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" +#include "../visitor/reset_visitor.hpp" +#include "../visitor/weight_size_visitor.hpp" + #include "layer_types.hpp" #include "add_merge.hpp" #include "sequential.hpp" @@ -21,7 +26,6 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -// TODO: refactor recurrent layer /** * This class implements the Recurrent Model for Visual Attention, using a * variety of possible layer implementations. @@ -39,16 +43,16 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class RecurrentAttention : public MultiLayer +class RecurrentAttention { public: /** @@ -78,7 +82,8 @@ class RecurrentAttention : public MultiLayer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -89,9 +94,10 @@ class RecurrentAttention : public MultiLayer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /* * Calculate the gradient using the output delta and the input activation. @@ -100,9 +106,41 @@ class RecurrentAttention : public MultiLayer * @param * (error) The calculated error. * @param * (gradient) The calculated gradient. */ - void Gradient(const InputType& /* input */, - const OutputType& /* error */, - OutputType& /* gradient */); + template + void Gradient(const arma::Mat& /* input */, + const arma::Mat& /* error */, + arma::Mat& /* gradient */); + + //! Get the model modules. + std::vector>& Model() { return network; } + + //! The value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } + + //! Get the parameters. + OutputDataType const& Parameters() const { return parameters; } + //! Modify the parameters. + OutputDataType& Parameters() { return parameters; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the module output size. + size_t OutSize() const { return outSize; } //! Get the number of steps to backpropagate through time. size_t const& Rho() const { return rho; } @@ -122,18 +160,20 @@ class RecurrentAttention : public MultiLayer // Gradient of the action module. if (backwardStep == (rho - 1)) { - actionModule->Gradient(initialInput, actionError, - actionModule->Gradient()); + boost::apply_visitor(GradientVisitor(initialInput, actionError), + actionModule); } else { - actionModule->Gradient(actionModule->OutputParameter(), actionError, - actionModule->Gradient()); + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, actionModule), actionError), + actionModule); } // Gradient of the recurrent module. - rnnModule->Gradient(rnnModule->OutputParameter(), recurrentError, - rnnModule->Gradient()); + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, rnnModule), recurrentError), + rnnModule); attentionGradient += intermediateGradient; } @@ -142,10 +182,10 @@ class RecurrentAttention : public MultiLayer size_t outSize; //! Locally-stored start module. - Layer* rnnModule; + LayerTypes<> rnnModule; //! Locally-stored input module. - Layer* actionModule; + LayerTypes<> actionModule; //! Number of steps to backpropagate through time (BPTT). size_t rho; @@ -156,38 +196,62 @@ class RecurrentAttention : public MultiLayer //! Locally-stored number of backward steps. size_t backwardStep; + //! If true dropout and scaling is disabled, see notes above. + bool deterministic; + //! Locally-stored weight object. - OutputType parameters; + OutputDataType parameters; //! Locally-stored model modules. - std::vector*> network; + std::vector> network; + + //! Locally-stored weight size visitor. + WeightSizeVisitor weightSizeVisitor; + + //! Locally-stored delta visitor. + DeltaVisitor deltaVisitor; + + //! Locally-stored output parameter visitor. + OutputParameterVisitor outputParameterVisitor; //! Locally-stored feedback output parameters. - std::vector feedbackOutputParameter; + std::vector feedbackOutputParameter; //! List of all module parameters for the backward pass (BBTT). - std::vector moduleOutputParameter; + std::vector moduleOutputParameter; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; //! Locally-stored recurrent error parameter. - OutputType recurrentError; + arma::mat recurrentError; //! Locally-stored action error parameter. - OutputType actionError; + arma::mat actionError; //! Locally-stored action delta. - OutputType actionDelta; + arma::mat actionDelta; //! Locally-stored recurrent delta. - OutputType rnnDelta; + arma::mat rnnDelta; //! Locally-stored initial action input. - InputType initialInput; + arma::mat initialInput; + + //! Locally-stored reset visitor. + ResetVisitor resetVisitor; //! Locally-stored attention gradient. - OutputType attentionGradient; + arma::mat attentionGradient; //! Locally-stored intermediate gradient for the attention module. - OutputType intermediateGradient; + arma::mat intermediateGradient; }; // class RecurrentAttention } // namespace ann diff --git a/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp b/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp new file mode 100644 index 0000000000..abc3da7727 --- /dev/null +++ b/src/mlpack/methods/ann/layer/recurrent_attention_impl.hpp @@ -0,0 +1,226 @@ +/** + * @file methods/ann/layer/recurrent_attention_impl.hpp + * @author Marcus Edel + * + * Implementation of the RecurrentAttention class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_RECURRENT_ATTENTION_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_RECURRENT_ATTENTION_IMPL_HPP + +// In case it hasn't yet been included. +#include "recurrent_attention.hpp" + +#include "../visitor/load_output_parameter_visitor.hpp" +#include "../visitor/save_output_parameter_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/forward_visitor.hpp" +#include "../visitor/gradient_set_visitor.hpp" +#include "../visitor/gradient_update_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +RecurrentAttention::RecurrentAttention() : + outSize(0), + rho(0), + forwardStep(0), + backwardStep(0), + deterministic(false) +{ + // Nothing to do. +} + +template +template +RecurrentAttention::RecurrentAttention( + const size_t outSize, + const RNNModuleType& rnn, + const ActionModuleType& action, + const size_t rho) : + outSize(outSize), + rnnModule(new RNNModuleType(rnn)), + actionModule(new ActionModuleType(action)), + rho(rho), + forwardStep(0), + backwardStep(0), + deterministic(false) +{ + network.push_back(rnnModule); + network.push_back(actionModule); +} + +template +template +void RecurrentAttention::Forward( + const arma::Mat& input, arma::Mat& output) +{ + // Initialize the action input. + if (initialInput.is_empty()) + { + initialInput = arma::zeros(outSize, input.n_cols); + } + + // Propagate through the action and recurrent module. + for (forwardStep = 0; forwardStep < rho; ++forwardStep) + { + if (forwardStep == 0) + { + boost::apply_visitor(ForwardVisitor(initialInput, + boost::apply_visitor(outputParameterVisitor, actionModule)), + actionModule); + } + else + { + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, rnnModule), boost::apply_visitor( + outputParameterVisitor, actionModule)), actionModule); + } + + // Initialize the glimpse input. + arma::mat glimpseInput = arma::zeros(input.n_elem, 2); + glimpseInput.col(0) = input; + glimpseInput.submat(0, 1, boost::apply_visitor(outputParameterVisitor, + actionModule).n_elem - 1, 1) = boost::apply_visitor( + outputParameterVisitor, actionModule); + + boost::apply_visitor(ForwardVisitor(glimpseInput, + boost::apply_visitor(outputParameterVisitor, rnnModule)), + rnnModule); + + // Save the output parameter when training the module. + if (!deterministic) + { + for (size_t l = 0; l < network.size(); ++l) + { + boost::apply_visitor(SaveOutputParameterVisitor( + moduleOutputParameter), network[l]); + } + } + } + + output = boost::apply_visitor(outputParameterVisitor, rnnModule); + + forwardStep = 0; + backwardStep = 0; +} + +template +template +void RecurrentAttention::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) +{ + if (intermediateGradient.is_empty() && backwardStep == 0) + { + // Initialize the attention gradients. + size_t weights = boost::apply_visitor(weightSizeVisitor, rnnModule) + + boost::apply_visitor(weightSizeVisitor, actionModule); + + intermediateGradient = arma::zeros(weights, 1); + attentionGradient = arma::zeros(weights, 1); + + // Initialize the action error. + actionError = arma::zeros( + boost::apply_visitor(outputParameterVisitor, actionModule).n_rows, + boost::apply_visitor(outputParameterVisitor, actionModule).n_cols); + } + + // Propagate the attention gradients. + if (backwardStep == 0) + { + size_t offset = 0; + offset += boost::apply_visitor(GradientSetVisitor( + intermediateGradient, offset), rnnModule); + boost::apply_visitor(GradientSetVisitor( + intermediateGradient, offset), actionModule); + + attentionGradient.zeros(); + } + + // Back-propagate through time. + for (; backwardStep < rho; backwardStep++) + { + if (backwardStep == 0) + { + recurrentError = gy; + } + else + { + recurrentError = actionDelta; + } + + for (size_t l = 0; l < network.size(); ++l) + { + boost::apply_visitor(LoadOutputParameterVisitor( + moduleOutputParameter), network[network.size() - 1 - l]); + } + + if (backwardStep == (rho - 1)) + { + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, actionModule), actionError, + actionDelta), actionModule); + } + else + { + boost::apply_visitor(BackwardVisitor(initialInput, actionError, + actionDelta), actionModule); + } + + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, rnnModule), recurrentError, rnnDelta), + rnnModule); + + if (backwardStep == 0) + { + g = rnnDelta.col(1); + } + else + { + g += rnnDelta.col(1); + } + + IntermediateGradient(); + } +} + +template +template +void RecurrentAttention::Gradient( + const arma::Mat& /* input */, + const arma::Mat& /* error */, + arma::Mat& /* gradient */) +{ + size_t offset = 0; + offset += boost::apply_visitor(GradientUpdateVisitor( + attentionGradient, offset), rnnModule); + boost::apply_visitor(GradientUpdateVisitor( + attentionGradient, offset), actionModule); +} + +template +template +void RecurrentAttention::serialize( + Archive& ar, const uint32_t /* version */) +{ + ar(CEREAL_NVP(rho)); + ar(CEREAL_NVP(outSize)); + ar(CEREAL_NVP(forwardStep)); + ar(CEREAL_NVP(backwardStep)); + + ar(CEREAL_VARIANT_POINTER(rnnModule)); + ar(CEREAL_VARIANT_POINTER(actionModule)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/recurrent_impl.hpp b/src/mlpack/methods/ann/layer/recurrent_impl.hpp new file mode 100644 index 0000000000..046c48fa0f --- /dev/null +++ b/src/mlpack/methods/ann/layer/recurrent_impl.hpp @@ -0,0 +1,361 @@ +/** + * @file methods/ann/layer/recurrent_impl.hpp + * @author Marcus Edel + * + * Implementation of the Recurrent class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_RECURRENT_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_RECURRENT_IMPL_HPP + +// In case it hasn't yet been included. +#include "recurrent.hpp" + +#include "../visitor/add_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" +#include "../visitor/gradient_zero_visitor.hpp" +#include "../visitor/input_shape_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +Recurrent::Recurrent() : + rho(0), + forwardStep(0), + backwardStep(0), + gradientStep(0), + deterministic(false), + ownsLayer(false) +{ + // Nothing to do. +} + +template +template< + typename StartModuleType, + typename InputModuleType, + typename FeedbackModuleType, + typename TransferModuleType +> +Recurrent::Recurrent( + const StartModuleType& start, + const InputModuleType& input, + const FeedbackModuleType& feedback, + const TransferModuleType& transfer, + const size_t rho) : + startModule(new StartModuleType(start)), + inputModule(new InputModuleType(input)), + feedbackModule(new FeedbackModuleType(feedback)), + transferModule(new TransferModuleType(transfer)), + rho(rho), + forwardStep(0), + backwardStep(0), + gradientStep(0), + deterministic(false), + ownsLayer(true) +{ + initialModule = new Sequential<>(); + mergeModule = new AddMerge<>(false, false, false); + recurrentModule = new Sequential<>(false, false); + + boost::apply_visitor(AddVisitor(inputModule), + initialModule); + boost::apply_visitor(AddVisitor(startModule), + initialModule); + boost::apply_visitor(AddVisitor(transferModule), + initialModule); + + boost::apply_visitor(AddVisitor(inputModule), mergeModule); + boost::apply_visitor(AddVisitor(feedbackModule), + mergeModule); + boost::apply_visitor(AddVisitor(mergeModule), + recurrentModule); + boost::apply_visitor(AddVisitor(transferModule), + recurrentModule); + + network.push_back(initialModule); + network.push_back(mergeModule); + network.push_back(feedbackModule); + network.push_back(recurrentModule); +} + +template +Recurrent::Recurrent( + const Recurrent& network) : + rho(network.rho), + forwardStep(network.forwardStep), + backwardStep(network.backwardStep), + gradientStep(network.gradientStep), + deterministic(network.deterministic), + ownsLayer(network.ownsLayer) +{ + startModule = boost::apply_visitor(copyVisitor, network.startModule); + inputModule = boost::apply_visitor(copyVisitor, network.inputModule); + feedbackModule = boost::apply_visitor(copyVisitor, network.feedbackModule); + transferModule = boost::apply_visitor(copyVisitor, network.transferModule); + initialModule = new Sequential<>(); + mergeModule = new AddMerge<>(false, false, false); + recurrentModule = new Sequential<>(false, false); + + boost::apply_visitor(AddVisitor(inputModule), + initialModule); + boost::apply_visitor(AddVisitor(startModule), + initialModule); + boost::apply_visitor(AddVisitor(transferModule), + initialModule); + + boost::apply_visitor(AddVisitor(inputModule), mergeModule); + boost::apply_visitor(AddVisitor(feedbackModule), + mergeModule); + boost::apply_visitor(AddVisitor(mergeModule), + recurrentModule); + boost::apply_visitor(AddVisitor(transferModule), + recurrentModule); + this->network.push_back(initialModule); + this->network.push_back(mergeModule); + this->network.push_back(feedbackModule); + this->network.push_back(recurrentModule); +} + +template +size_t +Recurrent::InputShape() const +{ + const size_t inputShapeStartModule = boost::apply_visitor(InShapeVisitor(), + startModule); + + // Return the input shape of the first module that we have. + if (inputShapeStartModule != 0) + { + return inputShapeStartModule; + } + // If input shape of first module is 0. + else + { + // Return input shape of the second module that we have. + const size_t inputShapeInputModule = boost::apply_visitor(InShapeVisitor(), + inputModule); + if (inputShapeInputModule != 0) + { + return inputShapeInputModule; + } + else // If the input shape of second module is 0. + { + // Return input shape of the third module that we have. + const size_t inputShapeFeedbackModule = boost::apply_visitor( + InShapeVisitor(), feedbackModule); + if (inputShapeFeedbackModule != 0) + { + return inputShapeFeedbackModule; + } + else // If the input shape of the third module is 0. + { + // Return the shape of the fourth module that we have. + const size_t inputShapeTransferModule = boost::apply_visitor( + InShapeVisitor(), transferModule); + if (inputShapeTransferModule != 0) + { + return inputShapeTransferModule; + } + else // If the input shape of the fourth module is 0. + { + return 0; + } + } + } + } +} + +template +template +void Recurrent::Forward( + const arma::Mat& input, arma::Mat& output) +{ + if (forwardStep == 0) + { + boost::apply_visitor(ForwardVisitor(input, output), initialModule); + } + else + { + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, inputModule)), + inputModule); + + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, transferModule), + boost::apply_visitor(outputParameterVisitor, feedbackModule)), + feedbackModule); + + boost::apply_visitor(ForwardVisitor(input, output), recurrentModule); + } + + output = boost::apply_visitor(outputParameterVisitor, transferModule); + + // Save the feedback output parameter when training the module. + if (!deterministic) + { + feedbackOutputParameter.push_back(output); + } + + forwardStep++; + if (forwardStep == rho) + { + forwardStep = 0; + backwardStep = 0; + + if (!recurrentError.is_empty()) + { + recurrentError.zeros(); + } + } +} + +template +template +void Recurrent::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) +{ + if (!recurrentError.is_empty()) + { + recurrentError += gy; + } + else + { + recurrentError = gy; + } + + if (backwardStep < (rho - 1)) + { + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, recurrentModule), recurrentError, + boost::apply_visitor(deltaVisitor, recurrentModule)), + recurrentModule); + + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, inputModule), + boost::apply_visitor(deltaVisitor, recurrentModule), g), + inputModule); + + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, feedbackModule), + boost::apply_visitor(deltaVisitor, recurrentModule), + boost::apply_visitor(deltaVisitor, feedbackModule)), feedbackModule); + } + else + { + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, initialModule), recurrentError, g), + initialModule); + } + + recurrentError = boost::apply_visitor(deltaVisitor, feedbackModule); + backwardStep++; +} + +template +template +void Recurrent::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */) +{ + if (gradientStep < (rho - 1)) + { + boost::apply_visitor(GradientVisitor(input, error), recurrentModule); + + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, mergeModule)), inputModule); + + boost::apply_visitor(GradientVisitor( + feedbackOutputParameter[feedbackOutputParameter.size() - 2 - + gradientStep], boost::apply_visitor(deltaVisitor, + mergeModule)), feedbackModule); + } + else + { + boost::apply_visitor(GradientZeroVisitor(), recurrentModule); + boost::apply_visitor(GradientZeroVisitor(), inputModule); + boost::apply_visitor(GradientZeroVisitor(), feedbackModule); + + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, startModule)), initialModule); + } + + gradientStep++; + if (gradientStep == rho) + { + gradientStep = 0; + feedbackOutputParameter.clear(); + } +} + +template +template +void Recurrent::serialize( + Archive& ar, const uint32_t /* version */) +{ + // Clean up memory, if we are loading. + if (cereal::is_loading()) + { + // Clear old things, if needed. + boost::apply_visitor(DeleteVisitor(), recurrentModule); + boost::apply_visitor(DeleteVisitor(), initialModule); + boost::apply_visitor(DeleteVisitor(), startModule); + network.clear(); + } + + ar(CEREAL_VARIANT_POINTER(startModule)); + ar(CEREAL_VARIANT_POINTER(inputModule)); + ar(CEREAL_VARIANT_POINTER(feedbackModule)); + ar(CEREAL_VARIANT_POINTER(transferModule)); + ar(CEREAL_NVP(rho)); + ar(CEREAL_NVP(ownsLayer)); + + // Set up the network. + if (cereal::is_loading()) + { + initialModule = new Sequential<>(); + mergeModule = new AddMerge<>(false, false, false); + recurrentModule = new Sequential<>(false, false); + + boost::apply_visitor(AddVisitor(inputModule), + initialModule); + boost::apply_visitor(AddVisitor(startModule), + initialModule); + boost::apply_visitor(AddVisitor(transferModule), + initialModule); + + boost::apply_visitor(AddVisitor(inputModule), + mergeModule); + boost::apply_visitor(AddVisitor(feedbackModule), + mergeModule); + boost::apply_visitor(AddVisitor(mergeModule), + recurrentModule); + boost::apply_visitor(AddVisitor(transferModule), + recurrentModule); + + network.push_back(initialModule); + network.push_back(mergeModule); + network.push_back(feedbackModule); + network.push_back(recurrentModule); + } +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/recurrent_layer.hpp b/src/mlpack/methods/ann/layer/recurrent_layer.hpp deleted file mode 100644 index 118bb2385e..0000000000 --- a/src/mlpack/methods/ann/layer/recurrent_layer.hpp +++ /dev/null @@ -1,107 +0,0 @@ -/** - * @file methods/ann/layer/recurrent_layer.hpp - * @author Ryan Curtin - * - * Base layer for recurrent neural network layers. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with the mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_RECURRENT_LAYER_HPP -#define MLPACK_METHODS_ANN_LAYER_RECURRENT_LAYER_HPP - -#include -#include "layer.hpp" - -namespace mlpack { -namespace ann { - -/** - * The `RecurrentLayer` provides a base layer for all layers that have recurrent - * functionality and store state between steps in a recurrent network. Any - * RecurrentLayer should only be used with a network type such as `RNN` that - * supports recurrent layers. - * - * Any recurrent layer that inherits from `RecurrentLayer` must implement the - * `ClearRecurrentState(bpttSteps, batchSize)` function; this function should - * allocate space to store previous states with the given batch size. See the - * documentation for that function for more details. - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. - */ -template -class RecurrentLayer : public Layer -{ - public: - /** - * Create the RecurrentLayer. - */ - RecurrentLayer(); - - // Virtual destructor is required for classes using inheritance. - virtual ~RecurrentLayer() { } - - //! Copy the given RecurrentLayer. - RecurrentLayer(const RecurrentLayer& other); - //! Take ownership of the given RecurrentLayer. - RecurrentLayer(RecurrentLayer&& other); - //! Copy the given RecurrentLayer. - RecurrentLayer& operator=(const RecurrentLayer& other); - //! Take ownership of the given RecurrentLayer. - RecurrentLayer& operator=(RecurrentLayer&& other); - - /** - * ClearRecurrentState() is called before any forward pass of a recurrent - * network. This function is responsible for allocating any memory necessary - * to store `bpttSteps` steps of previous forward and backward passes, with a - * batch size of `batchSize`. - * - * Any internal state of the recurrent layer should be set to 0. - */ - virtual void ClearRecurrentState( - const size_t bpttSteps, - const size_t batchSize) = 0; - - //! Get the current step index to use in a forward or backward pass. - size_t CurrentStep() const { return currentStep; } - //! Modify the current step index to use in a forward or backward pass. - //! (Don't do this inside of your recurrent layer's implementation! This is - //! meant to be done by the enclosing network.) - size_t& CurrentStep() { return currentStep; } - - //! Get the previous step index, representing the value of CurrentStep() in - //! the previous call to Forward() or Backward(). - size_t PreviousStep() const { return previousStep; } - //! Modify the previous step index, representing the value of CurrentStep() in - //! the previous call to Forward() or Backward(). (Don't modify this inside - //! of your recurrent layer's implementation! This is meant to be done by the - //! enclosing network.) - size_t& PreviousStep() { return previousStep; } - - //! If Forward() or Backward() has been called since ClearRecurrentState(), - //! this will return true. This should be used to determine if recurrent - //! state should be considered in computations. - bool HasPreviousStep() const { return previousStep != size_t(-1); } - - //! Serialize the recurrent layer. - template - void serialize(Archive& ar, const uint32_t /* version */); - - private: - //! The current index of the step. This is set by the enclosing network - //! during forward and backward passes. - size_t currentStep; - //! The previous index of the step. This is set by the enclosing network - //! during forward and backward passes. - size_t previousStep; -}; - -} // namespace ann -} // namespace mlpack - -#include "recurrent_layer_impl.hpp" - -#endif diff --git a/src/mlpack/methods/ann/layer/recurrent_layer_impl.hpp b/src/mlpack/methods/ann/layer/recurrent_layer_impl.hpp deleted file mode 100644 index 92eeb3da11..0000000000 --- a/src/mlpack/methods/ann/layer/recurrent_layer_impl.hpp +++ /dev/null @@ -1,84 +0,0 @@ -/** - * @file methods/ann/layer/recurrent_layer_impl.hpp - * @author Ryan Curtin - * - * Base layer for recurrent neural network layers. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with the mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_RECURRENT_LAYER_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_RECURRENT_LAYER_IMPL_HPP - -// In case it hasn't been included yet. -#include "recurrent_layer.hpp" - -namespace mlpack { -namespace ann { - -template -RecurrentLayer::RecurrentLayer() : - Layer(), - currentStep(0), - previousStep(0) -{ /* Nothing to do. */ } - -template -RecurrentLayer::RecurrentLayer(const RecurrentLayer& other) : - Layer(other), - currentStep(other.currentStep), - previousStep(other.previousStep) -{ /* Nothing else to do. */ } - -template -RecurrentLayer::RecurrentLayer(RecurrentLayer&& other) : - Layer(std::move(other)), - currentStep(std::move(other.currentStep)), - previousStep(std::move(other.previousStep)) -{ /* Nothing else to do. */ } - -template -RecurrentLayer& -RecurrentLayer::operator=(const RecurrentLayer& other) -{ - if (this != &other) - { - Layer::operator=(other); - currentStep = other.currentStep; - previousStep = other.previousStep; - } - - return *this; -} - -template -RecurrentLayer& -RecurrentLayer::operator=(RecurrentLayer&& other) -{ - if (this != &other) - { - Layer::operator=(std::move(other)); - currentStep = std::move(other.currentStep); - previousStep = std::move(other.previousStep); - } - - return *this; -} - -template -template -void RecurrentLayer::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_NVP(currentStep)); - ar(CEREAL_NVP(previousStep)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/reinforce_normal.hpp b/src/mlpack/methods/ann/layer/reinforce_normal.hpp similarity index 64% rename from src/mlpack/methods/ann/layer/not_adapted/reinforce_normal.hpp rename to src/mlpack/methods/ann/layer/reinforce_normal.hpp index 894c2b1c9f..5f5c8a00e8 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/reinforce_normal.hpp +++ b/src/mlpack/methods/ann/layer/reinforce_normal.hpp @@ -14,7 +14,6 @@ #define MLPACK_METHODS_ANN_LAYER_REINFORCE_NORMAL_HPP #include -#include "layer.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -23,16 +22,16 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the reinforce normal layer. The reinforce normal layer * implements the REINFORCE algorithm for the normal distribution. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class ReinforceNormalType : public Layer +class ReinforceNormal { public: /** @@ -40,10 +39,7 @@ class ReinforceNormalType : public Layer * * @param stdev Standard deviation used during the forward and backward pass. */ - ReinforceNormalType(const double stdev = 1.0); - - //! Clone the ReinforceNormalType object. This handles polymorphism correctly. - ReinforceNormalType* Clone() const { return new ReinforceNormalType(*this); } + ReinforceNormal(const double stdev = 1.0); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -52,7 +48,8 @@ class ReinforceNormalType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -63,17 +60,31 @@ class ReinforceNormalType : public Layer * @param * (gy) The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, - const OutputType& /* gy */, - OutputType& g); + template + void Backward(const DataType& input, const DataType& /* gy */, DataType& g); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } //! Get the value of the reward parameter. - double const& Reward() const { return reward; } + double Reward() const { return reward; } //! Modify the value of the deterministic parameter. double& Reward() { return reward; } //! Get the standard deviation used during forward and backward pass. - double const& StandardDeviation() const { return stdev; } + double StandardDeviation() const { return stdev; } /** * Serialize the layer @@ -88,15 +99,18 @@ class ReinforceNormalType : public Layer //! Locally-stored reward parameter. double reward; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Locally-stored output module parameter parameters. - std::vector moduleInputParameter; + std::vector moduleInputParameter; //! If true use maximum a posteriori during the forward pass. bool deterministic; -}; // class ReinforceNormalType. - -// Standard ReinforceNormal layer. -typedef ReinforceNormalType ReinforceNormal; +}; // class ReinforceNormal } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/reinforce_normal_impl.hpp b/src/mlpack/methods/ann/layer/reinforce_normal_impl.hpp similarity index 68% rename from src/mlpack/methods/ann/layer/not_adapted/reinforce_normal_impl.hpp rename to src/mlpack/methods/ann/layer/reinforce_normal_impl.hpp index d98820e5db..c2f92df476 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/reinforce_normal_impl.hpp +++ b/src/mlpack/methods/ann/layer/reinforce_normal_impl.hpp @@ -20,21 +20,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -ReinforceNormalType::ReinforceNormalType( +ReinforceNormal::ReinforceNormal( const double stdev) : stdev(stdev), reward(0.0), deterministic(false) { // Nothing to do here. } -template -void ReinforceNormalType::Forward( - const InputType& input, OutputType& output) +template +template +void ReinforceNormal::Forward( + const arma::Mat& input, arma::Mat& output) { if (!deterministic) { // Multiply by standard deviations and re-center the means to the mean. - output = arma::randn(input.n_rows, input.n_cols) * - stdev + input; + output = output.randn(input.n_rows, input.n_cols) * stdev + input; moduleInputParameter.push_back(input); } @@ -45,9 +45,10 @@ void ReinforceNormalType::Forward( } } -template -void ReinforceNormalType::Backward( - const InputType& input, const OutputType& /* gy */, OutputType& g) +template +template +void ReinforceNormal::Backward( + const DataType& input, const DataType& /* gy */, DataType& g) { g = (input - moduleInputParameter.back()) / std::pow(stdev, 2.0); @@ -58,13 +59,11 @@ void ReinforceNormalType::Backward( moduleInputParameter.pop_back(); } -template +template template -void ReinforceNormalType::serialize( +void ReinforceNormal::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(stdev)); } diff --git a/src/mlpack/methods/ann/layer/not_adapted/relu6.hpp b/src/mlpack/methods/ann/layer/relu6.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/relu6.hpp rename to src/mlpack/methods/ann/layer/relu6.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/relu6_impl.hpp b/src/mlpack/methods/ann/layer/relu6_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/not_adapted/relu6_impl.hpp rename to src/mlpack/methods/ann/layer/relu6_impl.hpp diff --git a/src/mlpack/methods/ann/layer/not_adapted/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp similarity index 54% rename from src/mlpack/methods/ann/layer/not_adapted/reparametrization.hpp rename to src/mlpack/methods/ann/layer/reparametrization.hpp index 833fa4c1a0..d3a183b9fd 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -15,15 +15,15 @@ #include -#include "layer.hpp" -// #include "../activation_functions/softplus_function.hpp" +#include "layer_types.hpp" +#include "../activation_functions/softplus_function.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Implementation of the Reparametrization layer class. This layer samples from - * the given parameters of a normal distribution. + * Implementation of the Reparametrization layer class. This layer samples from the + * given parameters of a normal distribution. * * This class also supports beta-VAE, a state-of-the-art framework for * automated discovery of interpretable factorised latent representations from @@ -38,77 +38,61 @@ namespace ann /** Artificial Neural Network. */ { * author = {Irina Higgins, Loic Matthey, Arka Pal, Christopher Burgess, * Xavier Glorot, Matthew Botvinick, Shakir Mohamed and * Alexander Lerchner | Google DeepMind}, - * journal = {2017 International Conference on Learning Representations - * (ICLR)}, + * journal = {2017 International Conference on Learning Representations(ICLR)}, * year = {2017}, * url = {https://deepmind.com/research/publications/beta-VAE-Learning-Basic-Visual-Concepts-with-a-Constrained-Variational-Framework} * } * @endcode * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class ReparametrizationType : public Layer +class Reparametrization { public: + //! Create the Reparametrization object. + Reparametrization(); + /** - * Create the Reparametrization layer object. Note that the inputs are - * expected to be the parameters of the normal distribution; see the - * documentation for Forward(). + * Create the Reparametrization layer object using the specified sample vector size. * + * @param latentSize The number of output latent units. * @param stochastic Whether we want random sample or constant. * @param includeKl Whether we want to include KL loss in backward function. * @param beta The beta (hyper)parameter for beta-VAE mentioned above. */ - ReparametrizationType(const bool stochastic = true, - const bool includeKl = true, - const double beta = 1); - - /** - * Clone the ReparametrizationType object. This handles polymorphism - * correctly. - */ - ReparametrizationType* Clone() const - { - return new ReparametrizationType(*this); - } - - // Virtual destructor. - virtual ~ReparametrizationType() { } + Reparametrization(const size_t latentSize, + const bool stochastic = true, + const bool includeKl = true, + const double beta = 1); //! Copy Constructor. - ReparametrizationType(const ReparametrizationType& layer); + Reparametrization(const Reparametrization& layer); //! Move Constructor. - ReparametrizationType(ReparametrizationType&& layer); + Reparametrization(Reparametrization&& layer); //! Copy assignment operator. - ReparametrizationType& operator=(const ReparametrizationType& layer); + Reparametrization& operator=(const Reparametrization& layer); //! Move assignment operator. - ReparametrizationType& operator=(ReparametrizationType&& layer); + Reparametrization& operator=(Reparametrization&& layer); /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. * - * Note that `input` is expected to be the parameters of the distribution. - * The first `input.n_rows / 2` elements correspond to the - * pre-standard-deviation values for each output element, and the second - * `input.n_rows / 2` elements correspond to the means for each element. - * Thus, the output size of the layer is the number of input elements divided - * by 2. - * * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -119,54 +103,60 @@ class ReparametrizationType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the output size. + size_t const& OutputSize() const { return latentSize; } + //! Modify the output size. + size_t& OutputSize() { return latentSize; } //! Get the KL divergence with standard normal. - double Loss(); + double Loss() + { + if (!includeKl) + return 0; + + return -0.5 * beta * arma::accu(2 * arma::log(stdDev) - arma::pow(stdDev, 2) + - arma::pow(mean, 2) + 1) / mean.n_cols; + } //! Get the value of the stochastic parameter. bool Stochastic() const { return stochastic; } - //! Modify the value of the stochastic parameter. - bool& Stochastic() { return stochastic; } //! Get the value of the includeKl parameter. bool IncludeKL() const { return includeKl; } - //! Modify the value of the includeKl parameter. - bool& IncludeKL() { return includeKl; } //! Get the value of the beta hyperparameter. double Beta() const { return beta; } - //! Modify the value of the beta hyperparameter. - double& Beta() { return beta; } - void ComputeOutputDimensions() + size_t InputShape() const { - const size_t inputElem = std::accumulate(this->inputDimensions.begin(), - this->inputDimensions.end(), 0); - if (inputElem % 2 != 0) - { - std::ostringstream oss; - oss << "Reparametrization layer requires that the total number of input " - << "elements is divisible by 2! (Received input with " << inputElem - << " total elements.)"; - throw std::invalid_argument(oss.str()); - } - - this->outputDimensions = std::vector( - this->inputDimensions.size(), 1); - // This flattens the input, and removes half the elements. - this->outputDimensions[0] = inputElem / 2; + return 2 * latentSize; } /** - * Serialize the layer. + * Serialize the layer */ template void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored number of output units. + size_t latentSize; + //! If false, sample will be constant. bool stochastic; @@ -176,22 +166,25 @@ class ReparametrizationType : public Layer //! The beta hyperparameter for constrained variational frameworks. double beta; + //! Locally-stored delta object. + OutputDataType delta; + //! Locally-stored current gaussian sample. - OutputType gaussianSample; + OutputDataType gaussianSample; //! Locally-stored current mean. - OutputType mean; + OutputDataType mean; //! Locally-stored pre standard deviation. //! After softplus activation gives standard deviation. - OutputType preStdDev; + OutputDataType preStdDev; //! Locally-stored current standard deviation. - OutputType stdDev; -}; // class ReparametrizationType + OutputDataType stdDev; -// Standard Reparametrization layer. -typedef ReparametrizationType Reparametrization; + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class Reparametrization } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp new file mode 100644 index 0000000000..117e67a620 --- /dev/null +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -0,0 +1,156 @@ +/** + * @file methods/ann/layer/reparametrization_impl.hpp + * @author Atharva Khandait + * + * Implementation of the Reparametrization layer class which samples from a + * gaussian distribution. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_IMPL_HPP + +// In case it hasn't yet been included. +#include "reparametrization.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +Reparametrization::Reparametrization() : + latentSize(0), + stochastic(true), + includeKl(true), + beta(1) +{ + // Nothing to do here. +} + +template +Reparametrization::Reparametrization( + const size_t latentSize, + const bool stochastic, + const bool includeKl, + const double beta) : + latentSize(latentSize), + stochastic(stochastic), + includeKl(includeKl), + beta(beta) +{ + if (includeKl == false && beta != 1) + { + Log::Info << "The beta parameter will be ignored as KL divergence is not " + << "included." << std::endl; + } +} + +template +Reparametrization::Reparametrization( + const Reparametrization& layer) : + latentSize(layer.latentSize), + stochastic(layer.stochastic), + includeKl(layer.includeKl), + beta(layer.beta) +{ + // Nothing to do here. +} + +template +Reparametrization::Reparametrization( + Reparametrization&& layer) : + latentSize(std::move(layer.latentSize)), + stochastic(std::move(layer.stochastic)), + includeKl(std::move(layer.includeKl)), + beta(std::move(layer.beta)) +{ + // Nothing to do here. +} + +template +Reparametrization& +Reparametrization:: +operator=(const Reparametrization& layer) +{ + if (this != &layer) + { + latentSize = layer.latentSize; + stochastic = layer.stochastic; + includeKl = layer.includeKl; + beta = layer.beta; + } + return *this; +} + +template +Reparametrization& +Reparametrization:: +operator=(Reparametrization&& layer) +{ + if (this != &layer) + { + latentSize = std::move(layer.latentSize); + stochastic = std::move(layer.stochastic); + includeKl = std::move(layer.includeKl); + beta = std::move(layer.beta); + } + return *this; +} + + +template +template +void Reparametrization::Forward( + const arma::Mat& input, arma::Mat& output) +{ + if (input.n_rows != 2 * latentSize) + { + Log::Fatal << "The output size of layer before the Reparametrization " + << "layer should be 2 * latent size of the Reparametrization layer!" + << std::endl; + } + + mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); + preStdDev = input.submat(0, 0, latentSize - 1, input.n_cols - 1); + + if (stochastic) + gaussianSample = arma::randn >(latentSize, input.n_cols); + else + gaussianSample = arma::ones >(latentSize, input.n_cols) * 0.7; + + SoftplusFunction::Fn(preStdDev, stdDev); + output = mean + stdDev % gaussianSample; +} + +template +template +void Reparametrization::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) +{ + SoftplusFunction::Deriv(preStdDev, g); + + if (includeKl) + { + g = join_cols(gy % std::move(gaussianSample) % g + (-1 / stdDev + stdDev) + % g * beta, gy + mean * beta / mean.n_cols); + } + else + g = join_cols(gy % std::move(gaussianSample) % g, gy); +} + +template +template +void Reparametrization::serialize( + Archive& ar, const uint32_t /* version */) +{ + ar(CEREAL_NVP(latentSize)); + ar(CEREAL_NVP(stochastic)); + ar(CEREAL_NVP(includeKl)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/select.hpp b/src/mlpack/methods/ann/layer/select.hpp similarity index 53% rename from src/mlpack/methods/ann/layer/not_adapted/select.hpp rename to src/mlpack/methods/ann/layer/select.hpp index 001991658f..f67d57e37f 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/select.hpp +++ b/src/mlpack/methods/ann/layer/select.hpp @@ -14,38 +14,31 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The select module selects the specified dimensions from a given input point. + * The select module selects the specified column from a given input matrix. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class SelectType : public Layer +class Select { public: /** * Create the Select object. * - * @param index The first dimension to extract from the input. - * @param elements The number of elements that should be used. If 0 is given, - * then all dimensions starting with index up to the number of dimensions - * are used. + * @param index The column which should be extracted from the given input. + * @param elements The number of elements that should be used. */ - SelectType(const size_t index = 0, const size_t elements = 0); - - //! Clone the SelectType object. This handles polymorphism correctly. - SelectType* Clone() const { return new SelectType(*this); } + Select(const size_t index = 0, const size_t elements = 0); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -54,7 +47,8 @@ class SelectType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -65,15 +59,26 @@ class SelectType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the column index. - const size_t& Index() const { return index; } + size_t const& Index() const { return index; } //! Get the number of elements selected. - const size_t& NumElements() const { return elements; } + size_t const& NumElements() const { return elements; } /** * Serialize the layer @@ -81,34 +86,19 @@ class SelectType : public Layer template void serialize(Archive& ar, const uint32_t /* version */); - const std::vector OutputDimensions() const - { - std::vector outputDimensions(inputDimensions.size(), 1); - if (elements > 0) - { - outputDimensions[0] = elements; - } - else - { - // Compute the total number of dimensions. - const size_t totalDims = std::accumulate(inputDimensions.begin(), - inputDimensions.end(), 0); - outputDimensions[0] = (totalDims - index); - } - - return outputDimensions; - } - private: //! Locally-stored column index. size_t index; //! Locally-stored number of elements selected. size_t elements; -}; // class SelectType -// Standard Select layer. -typedef SelectType Select; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class Select } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/select_impl.hpp b/src/mlpack/methods/ann/layer/select_impl.hpp similarity index 58% rename from src/mlpack/methods/ann/layer/not_adapted/select_impl.hpp rename to src/mlpack/methods/ann/layer/select_impl.hpp index 1d36b4d517..55baa59e8c 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/select_impl.hpp +++ b/src/mlpack/methods/ann/layer/select_impl.hpp @@ -18,8 +18,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -SelectType::SelectType( +template +Select::Select( const size_t index, const size_t elements) : index(index), @@ -28,44 +28,43 @@ SelectType::SelectType( // Nothing to do here. } -template -void SelectType::Forward( - const InputType& input, OutputType& output) +template +template +void Select::Forward( + const arma::Mat& input, arma::Mat& output) { if (elements == 0) { - output = input.rows(index, input.n_rows - 1); + output = input.col(index); } else { - output = input.rows(index, index + elements - 1); + output = input.submat(0, index, elements - 1, index); } } -template -void SelectType::Backward( - const InputType& /* input */, - const OutputType& gy, - OutputType& g) +template +template +void Select::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { - // TODO: not sure if this is right if (elements == 0) { g = gy; } else { - g = gy.rows(0, elements - 1); + g = gy.submat(0, 0, elements - 1, 0); } } -template +template template -void SelectType::serialize( +void Select::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(index)); ar(CEREAL_NVP(elements)); } diff --git a/src/mlpack/methods/ann/layer/sequential.hpp b/src/mlpack/methods/ann/layer/sequential.hpp new file mode 100644 index 0000000000..bd1857b1ba --- /dev/null +++ b/src/mlpack/methods/ann/layer/sequential.hpp @@ -0,0 +1,267 @@ +/** + * @file methods/ann/layer/sequential.hpp + * @author Marcus Edel + * + * Definition of the Sequential class, which acts as a feed-forward fully + * connected network container. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_SEQUENTIAL_HPP +#define MLPACK_METHODS_ANN_LAYER_SEQUENTIAL_HPP + +#include + +#include "../visitor/delete_visitor.hpp" +#include "../visitor/copy_visitor.hpp" +#include "../visitor/delta_visitor.hpp" +#include "../visitor/output_height_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" +#include "../visitor/output_width_visitor.hpp" +#include "../visitor/input_shape_visitor.hpp" + +#include "layer_types.hpp" +#include "add_merge.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Implementation of the Sequential class. The sequential class works as a + * feed-forward fully connected network container which plugs various layers + * together. + * + * This class can also be used as a container for a residual block. In that + * case, the sizes of the input and output matrices of this class should be + * equal. A typedef has been added for use as a Residual<> class. + * + * For more information, refer the following paper. + * + * @code + * @article{He15, + * author = {Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun}, + * title = {Deep Residual Learning for Image Recognition}, + * year = {2015}, + * url = {https://arxiv.org/abs/1512.03385}, + * eprint = {1512.03385}, + * } + * @endcode + * + * Note: If this class is used as the first layer of a network, it should be + * preceded by IdentityLayer<>. + * + * Note: This class should at least have two layers for a call to its Gradient() + * function. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam Residual If true, use the object as a Residual block. + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + bool Residual = false, + typename... CustomLayers +> +class Sequential +{ + public: + /** + * Create the Sequential object using the specified parameters. + * + * @param model Expose the all network modules. + */ + Sequential(const bool model = true); + + /** + * Create the Sequential object using the specified parameters. + * + * @param model Expose all the network modules. + * @param ownsLayers If true, then this module will delete its layers when + * deallocated. + */ + Sequential(const bool model, const bool ownsLayers); + + //! Copy constructor. + Sequential(const Sequential& layer); + + //! Copy assignment operator. + Sequential& operator = (const Sequential& layer); + + //! Destroy the Sequential object. + ~Sequential(); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const arma::Mat& input, arma::Mat& output); + + /** + * Ordinary feed backward pass of a neural network, using 3rd-order tensors as + * input, calculating the function f(x) by propagating x backwards through f. + * Using the results from the feed forward pass. + * + * @param * (input) The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); + + /* + * Calculate the gradient using the output delta and the input activation. + * + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. + */ + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */); + + /* + * Add a new module to the model. + * + * @param args The layer parameter. + */ + template + void Add(Args... args) { network.push_back(new LayerType(args...)); } + + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + + //! Return the model modules. + std::vector >& Model() + { + if (model) + { + return network; + } + + return empty; + } + + //! Return the initial point for the optimization. + const arma::mat& Parameters() const { return parameters; } + //! Modify the initial point for the optimization. + arma::mat& Parameters() { return parameters; } + + //! Get the input parameter. + arma::mat const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + arma::mat& InputParameter() { return inputParameter; } + + //! Get the output parameter. + arma::mat const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + arma::mat& OutputParameter() { return outputParameter; } + + //! Get the delta. + arma::mat const& Delta() const { return delta; } + //! Modify the delta. + arma::mat& Delta() { return delta; } + + //! Get the gradient. + arma::mat const& Gradient() const { return gradient; } + //! Modify the gradient. + arma::mat& Gradient() { return gradient; } + + size_t InputShape() const; + + /** + * Serialize the layer + */ + template + void serialize(Archive& ar, const uint32_t /* version */); + + private: + //! Parameter which indicates if the modules should be exposed. + bool model; + + //! Indicator if we already initialized the model. + bool reset; + + //! Locally-stored network modules. + std::vector > network; + + //! Locally-stored model parameters. + arma::mat parameters; + + //! Locally-stored delta visitor. + DeltaVisitor deltaVisitor; + + //! Locally-stored output parameter visitor. + OutputParameterVisitor outputParameterVisitor; + + //! Locally-stored delete visitor. + DeleteVisitor deleteVisitor; + + //! Locally-stored empty list of modules. + std::vector > empty; + + //! Locally-stored delta object. + arma::mat delta; + + //! Locally-stored input parameter object. + arma::mat inputParameter; + + //! Locally-stored output parameter object. + arma::mat outputParameter; + + //! Locally-stored gradient object. + arma::mat gradient; + + //! Locally-stored output width visitor. + OutputWidthVisitor outputWidthVisitor; + + //! Locally-stored output height visitor. + OutputHeightVisitor outputHeightVisitor; + + //! Locally-stored copy visitor + CopyVisitor copyVisitor; + + //! The input width. + size_t width; + + //! The input height. + size_t height; + + //! Whether we are responsible for deleting the layers held in this module. + bool ownsLayers; +}; // class Sequential + +/* + * Convenience typedef for use as Residual<> layer. + */ +template< + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename... CustomLayers +> +using Residual = Sequential< + InputDataType, OutputDataType, true, CustomLayers...>; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "sequential_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/sequential_impl.hpp b/src/mlpack/methods/ann/layer/sequential_impl.hpp new file mode 100644 index 0000000000..1290a15ebb --- /dev/null +++ b/src/mlpack/methods/ann/layer/sequential_impl.hpp @@ -0,0 +1,270 @@ +/** + * @file methods/ann/layer/sequential_impl.hpp + * @author Marcus Edel + * + * Implementation of the Sequential class, which acts as a feed-forward fully + * connected network container. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_SEQUENTIAL_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_SEQUENTIAL_IMPL_HPP + +// In case it hasn't yet been included. +#include "sequential.hpp" + +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" +#include "../visitor/set_input_height_visitor.hpp" +#include "../visitor/set_input_width_visitor.hpp" +#include "../visitor/input_shape_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +Sequential:: +Sequential(const bool model) : + model(model), reset(false), width(0), height(0), ownsLayers(!model) +{ + // Nothing to do here. +} + +template +Sequential:: +Sequential(const bool model, const bool ownsLayers) : + model(model), reset(false), width(0), height(0), ownsLayers(ownsLayers) +{ + // Nothing to do here. +} + +template +Sequential:: +Sequential(const Sequential& layer) : + model(layer.model), + reset(layer.reset), + width(layer.width), + height(layer.height), + ownsLayers(layer.ownsLayers) +{ + // Nothing to do here. +} + +template +Sequential& +Sequential:: +operator = (const Sequential& layer) +{ + if (this != &layer) + { + model = layer.model; + reset = layer.reset; + width = layer.width; + height = layer.height; + ownsLayers = layer.ownsLayers; + parameters = layer.parameters; + network.clear(); + // Build new layers according to source network. + for (size_t i = 0; i < layer.network.size(); ++i) + { + this->network.push_back(boost::apply_visitor(copyVisitor, + layer.network[i])); + } + } + return *this; +} + + +template +Sequential< + InputDataType, OutputDataType, Residual, CustomLayers...>::~Sequential() +{ + if (!model && ownsLayers) + { + for (LayerTypes& layer : network) + boost::apply_visitor(deleteVisitor, layer); + } +} + +template +size_t Sequential:: +InputShape() const +{ + size_t inputShape = 0; + + for (size_t l = 0; l < network.size(); ++l) + { + if (inputShape == 0) + inputShape = boost::apply_visitor(InShapeVisitor(), network[l]); + else + break; + } + + return inputShape; +} + +template +template +void Sequential:: +Forward(const arma::Mat& input, arma::Mat& output) +{ + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network.front())), + network.front()); + + if (!reset) + { + if (boost::apply_visitor(outputWidthVisitor, network.front()) != 0) + { + width = boost::apply_visitor(outputWidthVisitor, network.front()); + } + + if (boost::apply_visitor(outputHeightVisitor, network.front()) != 0) + { + height = boost::apply_visitor(outputHeightVisitor, network.front()); + } + } + + for (size_t i = 1; i < network.size(); ++i) + { + if (!reset) + { + // Set the input width. + boost::apply_visitor(SetInputWidthVisitor(width), network[i]); + + // Set the input height. + boost::apply_visitor(SetInputHeightVisitor(height), network[i]); + } + + boost::apply_visitor(ForwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[i - 1]), + boost::apply_visitor(outputParameterVisitor, network[i])), + network[i]); + + if (!reset) + { + // Get the output width. + if (boost::apply_visitor(outputWidthVisitor, network[i]) != 0) + { + width = boost::apply_visitor(outputWidthVisitor, network[i]); + } + + // Get the output height. + if (boost::apply_visitor(outputHeightVisitor, network[i]) != 0) + { + height = boost::apply_visitor(outputHeightVisitor, network[i]); + } + } + } + + if (!reset) + { + reset = true; + } + + output = boost::apply_visitor(outputParameterVisitor, network.back()); + + if (Residual) + { + if (arma::size(output) != arma::size(input)) + { + Log::Fatal << "The sizes of the output and input matrices of the Residual" + << " block should be equal. Please examine the network architecture." + << std::endl; + } + output += input; + } +} + +template +template +void Sequential< + InputDataType, OutputDataType, Residual, CustomLayers...>::Backward( + const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) +{ + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network.back()), gy, + boost::apply_visitor(deltaVisitor, network.back())), + network.back()); + + for (size_t i = 2; i < network.size() + 1; ++i) + { + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i])), + network[network.size() - i]); + } + + g = boost::apply_visitor(deltaVisitor, network.front()); + + if (Residual) + { + g += gy; + } +} + +template +template +void Sequential:: +Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& /* gradient */) +{ + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - 2]), error), + network.back()); + + for (size_t i = 2; i < network.size(); ++i) + { + boost::apply_visitor(GradientVisitor(boost::apply_visitor( + outputParameterVisitor, network[network.size() - i - 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1])), + network[network.size() - i]); + } + + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, network[1])), network.front()); +} + +template +template +void Sequential< + InputDataType, OutputDataType, Residual, CustomLayers...>::serialize( + Archive& ar, const uint32_t /* version */) +{ + // If loading, delete the old layers. + if (cereal::is_loading()) + { + for (LayerTypes& layer : network) + { + boost::apply_visitor(deleteVisitor, layer); + } + } + + ar(CEREAL_NVP(model)); + ar(CEREAL_VECTOR_VARIANT_POINTER(network)); + + ar(CEREAL_NVP(ownsLayers)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp deleted file mode 100644 index 5e13d2a9b4..0000000000 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/** - * @file serialization.hpp - * @author Ryan Curtin - * - * Set up polymorphic serialization correctly for layer types. If you need - * custom serialization for a non-standard type, you will have to use the macros - * in this file. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_SERIALIZATION_HPP -#define MLPACK_METHODS_ANN_LAYER_SERIALIZATION_HPP - -#define CEREAL_REGISTER_MLPACK_LAYERS(...) \ - CEREAL_REGISTER_TYPE(mlpack::ann::Layer<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::MultiLayer<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::RecurrentLayer<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::AddType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::AlphaDropoutType<__VA_ARGS__>); \ - /* Base layers from base_layer.hpp. */ \ - CEREAL_REGISTER_TYPE(mlpack::ann::SigmoidType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::ReLUType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::TanHType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::SoftPlusType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::HardSigmoidType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::SwishType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::MishType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::LiSHTType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::GELUType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::ElliotType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::ElishType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::GaussianType<__VA_ARGS__>); \ - /* (end of base_layer.hpp) */ \ - CEREAL_REGISTER_TYPE(mlpack::ann::ConcatenateType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::ConvolutionType< \ - mlpack::ann::NaiveConvolution, \ - mlpack::ann::NaiveConvolution, \ - mlpack::ann::NaiveConvolution, \ - __VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::DropConnectType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::DropoutType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::LeakyReLUType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::Linear3DType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::LinearType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::LinearNoBiasType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::LogSoftMaxType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::LSTMType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::MaxPoolingType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::NoisyLinearType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::PaddingType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::RBFType<__VA_ARGS__>); \ - -CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); - -#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/softmax.hpp b/src/mlpack/methods/ann/layer/softmax.hpp similarity index 58% rename from src/mlpack/methods/ann/layer/not_adapted/softmax.hpp rename to src/mlpack/methods/ann/layer/softmax.hpp index 18f2cd9db6..43fd0856c9 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/softmax.hpp +++ b/src/mlpack/methods/ann/layer/softmax.hpp @@ -16,8 +16,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -28,21 +26,22 @@ namespace ann /** Artificial Neural Network. */ { * numbers. It should be used for inference only and not with NLL loss (use * LogSoftMax instead). * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class SoftmaxType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class Softmax { public: - //! Create the Softmax object. - SoftmaxType(); - - //! Clone the SoftmaxType object. This handles polymorphism correctly. - SoftmaxType* Clone() const { return new SoftmaxType(*this); } + /** + * Create the Softmax object. + */ + Softmax(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -51,6 +50,7 @@ class SoftmaxType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -62,19 +62,37 @@ class SoftmaxType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); - //! Serialize the layer. + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the size of the weights. + size_t WeightSize() const { return 0; } + + //! Get the delta. + InputDataType& Delta() const { return delta; } + //! Modify the delta. + InputDataType& Delta() { return delta; } + + /** + * Serialize the layer. + */ template - void serialize(Archive& ar, const uint32_t /* version */); + void serialize(Archive& /* ar */, const uint32_t /* version */); private: -}; // class SoftmaxType + //! Locally-stored delta object. + OutputDataType delta; -// Convenience typedefs. - -// Standard Linear layer using no regularization. -typedef SoftmaxType Softmax; + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class Softmax } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/softmax_impl.hpp b/src/mlpack/methods/ann/layer/softmax_impl.hpp similarity index 65% rename from src/mlpack/methods/ann/layer/not_adapted/softmax_impl.hpp rename to src/mlpack/methods/ann/layer/softmax_impl.hpp index 34fbed04e4..500e0e108e 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/softmax_impl.hpp +++ b/src/mlpack/methods/ann/layer/softmax_impl.hpp @@ -19,14 +19,15 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -SoftmaxType::SoftmaxType() +template +Softmax::Softmax() { // Nothing to do here. } +template template -void SoftmaxType::Forward( +void Softmax::Forward( const InputType& input, OutputType& output) { @@ -35,22 +36,23 @@ void SoftmaxType::Forward( output = softmaxInput.each_row() / sum(softmaxInput, 0); } -template -void SoftmaxType::Backward( - const InputType& input, - const OutputType& gy, - OutputType& g) +template +template +void Softmax::Backward( + const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { g = input % (gy - arma::repmat(arma::sum(gy % input), input.n_rows, 1)); } -template +template template -void SoftmaxType::serialize( - Archive& ar, +void Softmax::serialize( + Archive& /* ar */, const uint32_t /* version */) { - ar(cereal::base_class>(this)); + // Nothing to do here. } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/softmin.hpp b/src/mlpack/methods/ann/layer/softmin.hpp similarity index 57% rename from src/mlpack/methods/ann/layer/not_adapted/softmin.hpp rename to src/mlpack/methods/ann/layer/softmin.hpp index ffb8c2e43d..57809992c2 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/softmin.hpp +++ b/src/mlpack/methods/ann/layer/softmin.hpp @@ -15,8 +15,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -24,22 +22,23 @@ namespace ann /** Artificial Neural Network. */ { * Implementation of the Softmin layer. The Softmin function takes as a input * a vector of K real numbers, rescaling them so that the elements of the * K-dimensional output vector lie in the range [0, 1] and sum to 1. - * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class SoftminType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class Softmin { public: - //! Create the Softmin object. - SoftminType(); - - //! Clone the SoftminType object. This handles polymorphism correctly. - SoftminType* Clone() const { return new SoftminType(*this); } + /** + * Create the Softmin object. + */ + Softmin(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -48,6 +47,7 @@ class SoftminType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output); /** @@ -59,18 +59,34 @@ class SoftminType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); - //! Serialize the layer. + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + InputDataType& Delta() const { return delta; } + //! Modify the delta. + InputDataType& Delta() { return delta; } + + /** + * Serialize the layer. + */ template - void serialize(Archive& ar, const uint32_t /* version */); -}; // class SoftminType + void serialize(Archive& /* ar */, const uint32_t /* version */); -// Convenience typedefs. - -// Standard Softmin layer using no regularization. -typedef SoftminType Softmin; + private: + //! Locally-stored delta object. + OutputDataType delta; + //! Locally stored output parameter object. + OutputDataType outputParameter; +}; // class Softmin } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/softmin_impl.hpp b/src/mlpack/methods/ann/layer/softmin_impl.hpp similarity index 65% rename from src/mlpack/methods/ann/layer/not_adapted/softmin_impl.hpp rename to src/mlpack/methods/ann/layer/softmin_impl.hpp index 4e200f2943..6945256b44 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/softmin_impl.hpp +++ b/src/mlpack/methods/ann/layer/softmin_impl.hpp @@ -18,14 +18,15 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -SoftminType::SoftminType() +template +Softmin::Softmin() { // Nothing to do here. } +template template -void SoftminType::Forward( +void Softmin::Forward( const InputType& input, OutputType& output) { @@ -34,22 +35,23 @@ void SoftminType::Forward( output = softminInput.each_row() / sum(softminInput, 0); } -template -void SoftminType::Backward( - const InputType& input, - const OutputType& gy, - OutputType& g) +template +template +void Softmin::Backward( + const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g) { g = input % (gy - arma::repmat(arma::sum(gy % input), input.n_rows, 1)); } -template +template template -void SoftminType::serialize( - Archive& ar, +void Softmin::serialize( + Archive& /* ar */, const uint32_t /* version */) { - ar(cereal::base_class>(this)); + // Nothing to do here. } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/softshrink.hpp b/src/mlpack/methods/ann/layer/softshrink.hpp similarity index 66% rename from src/mlpack/methods/ann/layer/not_adapted/softshrink.hpp rename to src/mlpack/methods/ann/layer/softshrink.hpp index ac3f11319c..93db85f55b 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/softshrink.hpp +++ b/src/mlpack/methods/ann/layer/softshrink.hpp @@ -18,8 +18,6 @@ #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artifical Neural Network. */ { @@ -45,25 +43,26 @@ namespace ann /** Artifical Neural Network. */ { * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ -template -class SoftShrinkType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class SoftShrink { public: /** - * Create SoftShrink object using specified hyperparameter lambda. + * Create Soft Shrink object using specified hyperparameter lambda. * * @param lambda The noise level of an image depends on settings of an - * imaging device. The settings can be used to select appropriate - * parameters for denoising methods. It is proportional to the noise - * level entered by the user. And it is calculated by multiplying the - * noise level sigma of the input(noisy image) and a coefficient 'a' - * which is one of the training parameters. Default value of lambda - * is 0.5. + * imaging device. The settings can be used to select appropriate + * parameters for denoising methods. It is proportional to the noise + * level entered by the user. + * And it is calculated by multiplying the + * noise level sigma of the input(noisy image) and a + * coefficient 'a' which is one of the training parameters. + * Default value of lambda is 0.5. */ - SoftShrinkType(const double lambda = 0.5); - - //! Clone the SoftShrinkType object. This handles polymorphism correctly. - SoftShrinkType* Clone() const { return new SoftShrinkType(*this); } + SoftShrink(const double lambda = 0.5); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -72,6 +71,7 @@ class SoftShrinkType : public Layer * @param input Input data used for evaluating the Soft Shrink function. * @param output Resulting output activation */ + template void Forward(const InputType& input, OutputType& output); /** @@ -83,26 +83,42 @@ class SoftShrinkType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const DataType& input, + DataType& gy, + DataType& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the hyperparameter lambda. double const& Lambda() const { return lambda; } //! Modify the hyperparameter lambda. double& Lambda() { return lambda; } - //! Serialize the layer. + /** + * Serialize the layer. + */ template void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Locally-stored hyperparamater lambda. double lambda; -}; // class SoftShrinkType - -// Convenience typedefs. - -// Standard SoftShrink layer. -typedef SoftShrinkType SoftShrink; +}; // class SoftShrink } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/softshrink_impl.hpp b/src/mlpack/methods/ann/layer/softshrink_impl.hpp similarity index 59% rename from src/mlpack/methods/ann/layer/not_adapted/softshrink_impl.hpp rename to src/mlpack/methods/ann/layer/softshrink_impl.hpp index 80358de911..f4be4b1136 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/softshrink_impl.hpp +++ b/src/mlpack/methods/ann/layer/softshrink_impl.hpp @@ -20,36 +20,38 @@ namespace ann /** Artificial Neural Network. */ { // This constructor is called for Soft Shrink activation function. // lambda is a hyperparameter. -template -SoftShrinkType::SoftShrinkType(const double lambda) : +template +SoftShrink::SoftShrink(const double lambda) : lambda(lambda) { // Nothing to do here. } +template template -void SoftShrinkType::Forward( +void SoftShrink::Forward( const InputType& input, OutputType& output) { - output = (input > lambda) % (input - lambda) + - (input < -lambda) % (input + lambda); + output = (input > lambda) % (input - lambda) + ( + input < -lambda) % (input + lambda); } -template -void SoftShrinkType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) +template +template +void SoftShrink::Backward( + const DataType& input, DataType& gy, DataType& g) { - g = gy % (arma::ones(arma::size(input)) - (input == 0)); + DataType derivative; + derivative = (arma::ones(arma::size(input)) - (input == 0)); + g = gy % derivative; } -template +template template -void SoftShrinkType::serialize( +void SoftShrink::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(lambda)); } diff --git a/src/mlpack/methods/ann/layer/not_adapted/spatial_dropout.hpp b/src/mlpack/methods/ann/layer/spatial_dropout.hpp similarity index 66% rename from src/mlpack/methods/ann/layer/not_adapted/spatial_dropout.hpp rename to src/mlpack/methods/ann/layer/spatial_dropout.hpp index 0fe1c08ef3..e44388b97d 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/spatial_dropout.hpp +++ b/src/mlpack/methods/ann/layer/spatial_dropout.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/spatial_dropout.hpp * @author Anjishnu Mukherjee @@ -16,12 +15,9 @@ #include #include -#include "layer.hpp" - namespace mlpack { namespace ann /** Artificial Neural Network. */ { -// TODO: this could likely use inputDimensions to remove the `size` parameter! /** * Implementation of the SpatialDropout layer. * @@ -40,29 +36,27 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam InputType The type of the layer's inputs. The layer automatically - * cast inputs to this type (Default: arma::mat). - * @tparam OutputType The type of the computation which also causes the output - * to also be in this type. The type also allows the computation and weight - * type to differ from the input type (Default: arma::mat). + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class SpatialDropoutType : public Layer +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class SpatialDropout { public: //! Create the SpatialDropout object. - SpatialDropoutType(); - + SpatialDropout(); /** * Create the SpatialDropout object using the specified parameters. * * @param size The number of channels of each input image. * @param ratio The probability of each channel getting dropped. */ - SpatialDropoutType(const size_t size, const double ratio = 0.5); - - //! Clone the SpatialDropoutType object. This handles polymorphism correctly. - SpatialDropoutType* Clone() const { return new SpatialDropoutType(*this); } + SpatialDropout(const size_t size, const double ratio = 0.5); /** * Ordinary feed forward pass of the SpatialDropout layer. @@ -70,7 +64,8 @@ class SpatialDropoutType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of the SpatialDropout layer. @@ -79,7 +74,20 @@ class SpatialDropoutType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the number of channels. size_t Size() const { return size; } @@ -87,6 +95,11 @@ class SpatialDropoutType : public Layer //! Modify the number of channels. size_t& Size() { return size; } + //! Get the value of the deterministic parameter. + bool Deterministic() const { return deterministic; } + //! Modify the value of the deterministic parameter. + bool& Deterministic() { return deterministic; } + //! Get the probability value. double Ratio() const { return ratio; } @@ -97,13 +110,21 @@ class SpatialDropoutType : public Layer scale = 1.0 / (1.0 - ratio); } - //! Serialize the layer. + /** + * Serialize the layer. + */ template void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Locally-stored mast object. - OutputType mask; + OutputDataType mask; //! The number of channels of each input image. size_t size; @@ -122,13 +143,11 @@ class SpatialDropoutType : public Layer //! The number of pixels in each feature map. size_t inputSize; + + //! If true dropout and scaling are disabled. + bool deterministic; }; // class SpatialDropout -// Convenience typedefs. - -// Standard SpatialDropout layer. -typedef SpatialDropoutType SpatialDropout; - } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/spatial_dropout_impl.hpp b/src/mlpack/methods/ann/layer/spatial_dropout_impl.hpp similarity index 57% rename from src/mlpack/methods/ann/layer/not_adapted/spatial_dropout_impl.hpp rename to src/mlpack/methods/ann/layer/spatial_dropout_impl.hpp index d8ae24010b..4bd2cb767b 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/spatial_dropout_impl.hpp +++ b/src/mlpack/methods/ann/layer/spatial_dropout_impl.hpp @@ -18,20 +18,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -SpatialDropoutType::SpatialDropoutType() : +template +SpatialDropout::SpatialDropout() : size(0), ratio(0.5), scale(1.0 / (1.0 - ratio)), reset(false), batchSize(0), - inputSize(0) + inputSize(0), + deterministic(false) { // Nothing to do here. } -template -SpatialDropoutType::SpatialDropoutType( +template +SpatialDropout::SpatialDropout( const size_t size, const double ratio) : size(size), @@ -39,14 +40,16 @@ SpatialDropoutType::SpatialDropoutType( scale(1.0 / (1.0 - ratio)), reset(false), batchSize(0), - inputSize(0) + inputSize(0), + deterministic(false) { // Nothing to do here. } -template -void SpatialDropoutType::Forward( - const InputType& input, OutputType& output) +template +template +void SpatialDropout::Forward( + const arma::Mat& input, arma::Mat& output) { Log::Assert(input.n_rows % size == 0, "Input features must be divisible \ by feature maps."); @@ -59,20 +62,16 @@ void SpatialDropoutType::Forward( } if (deterministic) - { output = input; - } else { output.zeros(arma::size(input)); - arma::Cube inputTemp( - const_cast(input).memptr(), inputSize, size, batchSize, - false, true); - arma::Cube outputTemp( - const_cast(output).memptr(), inputSize, size, batchSize, - false, true); - OutputType probabilities(1, size); - OutputType maskRow(1, size); + arma::cube inputTemp(const_cast(input).memptr(), inputSize, + size, batchSize, false, false); + arma::cube outputTemp(const_cast(output).memptr(), inputSize, + size, batchSize, false, false); + arma::mat probabilities(1, size); + arma::mat maskRow(1, size); probabilities.fill(ratio); ann::BernoulliDistribution<> bernoulli_dist(probabilities, false); maskRow = bernoulli_dist.Sample(); @@ -83,39 +82,36 @@ void SpatialDropoutType::Forward( } } -template -void SpatialDropoutType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) +template +template +void SpatialDropout::Backward( + const arma::Mat& input, const arma::Mat& gy, arma::Mat& g) { g.zeros(arma::size(input)); - arma::Cube gyTemp( - const_cast(gy).memptr(), inputSize, size, batchSize, false, - true); - arma::Cube gTemp( - const_cast(g).memptr(), inputSize, size, batchSize, false, - true); + arma::cube gyTemp(const_cast(gy).memptr(), inputSize, size, + batchSize, false, false); + arma::cube gTemp(const_cast(g).memptr(), inputSize, size, + batchSize, false, false); for (size_t n = 0; n < batchSize; n++) gTemp.slice(n) = gyTemp.slice(n) % mask * scale; } -template +template template -void SpatialDropoutType::serialize( +void SpatialDropout::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(size)); ar(CEREAL_NVP(ratio)); ar(CEREAL_NVP(batchSize)); ar(CEREAL_NVP(inputSize)); ar(CEREAL_NVP(reset)); + ar(CEREAL_NVP(deterministic)); // Reset scale. - if (Archive::is_loading::value) - scale = 1.0 / (1.0 - ratio); + scale = 1.0 / (1.0 - ratio); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/not_adapted/subview.hpp b/src/mlpack/methods/ann/layer/subview.hpp similarity index 69% rename from src/mlpack/methods/ann/layer/not_adapted/subview.hpp rename to src/mlpack/methods/ann/layer/subview.hpp index 77a8a85b3a..bc99b0cc44 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/subview.hpp +++ b/src/mlpack/methods/ann/layer/subview.hpp @@ -13,8 +13,7 @@ #define MLPACK_METHODS_ANN_LAYER_SUBVIEW_HPP #include - -#include "layer.hpp" +#include namespace mlpack { namespace ann { @@ -23,31 +22,34 @@ namespace ann { * Implementation of the subview layer. The subview layer modifies the input to * a submatrix of required size. * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class SubviewType : public Layer +class Subview { public: /** * Create the Subview layer object using the specified range of input to * accept. * + * @param inSize Width of sample. * @param beginRow Starting row index. * @param endRow Ending row index. * @param beginCol Starting column index. * @param endCol Ending column index. */ - SubviewType(const size_t beginRow = 0, - const size_t endRow = 0, - const size_t beginCol = 0, - const size_t endCol = 0) : + Subview(const size_t inSize = 1, + const size_t beginRow = 0, + const size_t endRow = 0, + const size_t beginCol = 0, + const size_t endCol = 0) : + inSize(inSize), beginRow(beginRow), endRow(endRow), beginCol(beginCol), @@ -56,9 +58,6 @@ class SubviewType : public Layer /* Nothing to do here */ } - //! Clone the SubviewType object. This handles polymorphism correctly. - SubviewType* Clone() const { return new SubviewType(*this); } - /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. @@ -66,18 +65,17 @@ class SubviewType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ + template void Forward(const InputType& input, OutputType& output) { - size_t batchSize = input.n_cols; + size_t batchSize = input.n_cols / inSize; // Check if subview parameters are within the indices of input sample. - // TODO: this seems incorrect - endRow = ((endRow < inputDimensions[0]) && (endRow >= beginRow))? - endRow : (inputDimensions[0] - 1); - endCol = ((endCol < inputDimensions[1]) && (endCol >= beginCol)) ? - endCol : (inputDimensions[1] - 1); + endRow = ((endRow < input.n_rows) && (endRow >= beginRow))? + endRow : (input.n_rows - 1); + endCol = ((endCol < inSize) && (endCol >= beginCol)) ? + endCol : (inSize - 1); - // TODO: this is maybe not right? output.set_size( (endRow - beginRow + 1) * (endCol - beginCol + 1), batchSize); @@ -113,13 +111,27 @@ class SubviewType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g) + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g) { g = gy; } + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the width of each sample. + size_t InSize() const { return inSize; } + //! Get the starting row index of subview vector or matrix. size_t const& BeginRow() const { return beginRow; } //! Modify the width of each sample. @@ -140,33 +152,13 @@ class SubviewType : public Layer //! Modify the width of each sample. size_t& EndCol() { return endCol; } - const std::vector OutputDimensions() const - { - // TODO: relax this restriction - for (size_t i = 2; i < inputDimensions.size(); ++i) - { - if (inputDimensions[i] > 1) - { - throw std::invalid_argument("Subview(): layer input must be two-" - "dimensional!"); - } - } - - std::vector outputDimensions(inputDimensions); - outputDimensions[0] = (endRow - beginRow + 1); - outputDimensions[1] = (endCol - beginCol + 1); - - return outputDimensions; - } - /** * Serialize the layer. */ template void serialize(Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - + ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(beginRow)); ar(CEREAL_NVP(endRow)); ar(CEREAL_NVP(beginCol)); @@ -174,6 +166,9 @@ class SubviewType : public Layer } private: + //! Width of each sample. + size_t inSize; + //! Starting row index of subview vector or matrix. size_t beginRow; @@ -185,10 +180,13 @@ class SubviewType : public Layer //! Ending column index of subview vector or matrix. size_t endCol; -}; // class SubviewType -// Standard Subview layer. -typedef SubviewType Subview; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class Subview } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/transposed_convolution.hpp b/src/mlpack/methods/ann/layer/transposed_convolution.hpp similarity index 72% rename from src/mlpack/methods/ann/layer/not_adapted/transposed_convolution.hpp rename to src/mlpack/methods/ann/layer/transposed_convolution.hpp index f3103d492b..f637ca3355 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/transposed_convolution.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution.hpp @@ -1,4 +1,3 @@ -// Temporarily drop. /** * @file methods/ann/layer/transposed_convolution.hpp * @author Shikhar Jaiswal @@ -22,7 +21,7 @@ #include #include -#include "layer.hpp" +#include "layer_types.hpp" #include "padding.hpp" namespace mlpack { @@ -35,23 +34,23 @@ namespace ann /** Artificial Neural Network. */ { * @tparam ForwardConvolutionRule Convolution to perform forward process. * @tparam BackwardConvolutionRule Convolution to perform backward process. * @tparam GradientConvolutionRule Convolution to calculate gradient. - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < typename ForwardConvolutionRule = NaiveConvolution, typename BackwardConvolutionRule = NaiveConvolution, typename GradientConvolutionRule = NaiveConvolution, - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class TransposedConvolutionType : public Layer +class TransposedConvolution { public: //! Create the Transposed Convolution object. - TransposedConvolutionType(); + TransposedConvolution(); /** * Create the Transposed Convolution object using the specified number of @@ -77,20 +76,19 @@ class TransposedConvolutionType : public Layer * @param outputHeight The height of the output data. * @param paddingType The type of padding (Valid or Same). Defaults to None. */ - // TODO: remove inputWidth and inputHeight? - TransposedConvolutionType(const size_t inSize, - const size_t outSize, - const size_t kernelWidth, - const size_t kernelHeight, - const size_t strideWidth = 1, - const size_t strideHeight = 1, - const size_t padW = 0, - const size_t padH = 0, - const size_t inputWidth = 0, - const size_t inputHeight = 0, - const size_t outputWidth = 0, - const size_t outputHeight = 0, - const std::string& paddingType = "None"); + TransposedConvolution(const size_t inSize, + const size_t outSize, + const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth = 1, + const size_t strideHeight = 1, + const size_t padW = 0, + const size_t padH = 0, + const size_t inputWidth = 0, + const size_t inputHeight = 0, + const size_t outputWidth = 0, + const size_t outputHeight = 0, + const std::string& paddingType = "None"); /** * Create the Transposed Convolution object using the specified number of @@ -120,24 +118,24 @@ class TransposedConvolutionType : public Layer * @param outputHeight The height of the output data. * @param paddingType The type of padding (Valid or Same). Defaults to None. */ - TransposedConvolutionType(const size_t inSize, - const size_t outSize, - const size_t kernelWidth, - const size_t kernelHeight, - const size_t strideWidth, - const size_t strideHeight, - const std::tuple& padW, - const std::tuple& padH, - const size_t inputWidth = 0, - const size_t inputHeight = 0, - const size_t outputWidth = 0, - const size_t outputHeight = 0, - const std::string& paddingType = "None"); + TransposedConvolution(const size_t inSize, + const size_t outSize, + const size_t kernelWidth, + const size_t kernelHeight, + const size_t strideWidth, + const size_t strideHeight, + const std::tuple& padW, + const std::tuple& padH, + const size_t inputWidth = 0, + const size_t inputHeight = 0, + const size_t outputWidth = 0, + const size_t outputHeight = 0, + const std::string& paddingType = "None"); /* * Set the weight and bias term. */ - void SetWeights(const typename OutputType::elem_type* weightsPtr); + void Reset(); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -146,7 +144,8 @@ class TransposedConvolutionType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -157,122 +156,135 @@ class TransposedConvolutionType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); - /** + /* * Calculate the gradient using the output delta and the input activation. * * @param * (input) The input parameter used for calculating the gradient. * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& /* input */, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } //! Get the weight of the layer. - arma::Cube const& Weight() const - { - return weight; - } + arma::cube const& Weight() const { return weight; } //! Modify the weight of the layer. - arma::Cube& Weight() { return weight; } + arma::cube& Weight() { return weight; } //! Get the bias of the layer. - OutputType const& Bias() const { return bias; } + arma::mat const& Bias() const { return bias; } //! Modify the bias of the layer. - OutputType& Bias() { return bias; } + arma::mat& Bias() { return bias; } + + //! Get the input parameter. + InputDataType const& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the input width. - size_t const& InputWidth() const { return inputWidth; } + size_t InputWidth() const { return inputWidth; } //! Modify input the width. size_t& InputWidth() { return inputWidth; } //! Get the input height. - size_t const& InputHeight() const { return inputHeight; } + size_t InputHeight() const { return inputHeight; } //! Modify the input height. size_t& InputHeight() { return inputHeight; } //! Get the output width. - size_t const& OutputWidth() const { return outputWidth; } + size_t OutputWidth() const { return outputWidth; } //! Modify the output width. size_t& OutputWidth() { return outputWidth; } //! Get the output height. - size_t const& OutputHeight() const { return outputHeight; } + size_t OutputHeight() const { return outputHeight; } //! Modify the output height. size_t& OutputHeight() { return outputHeight; } //! Get the input size. - size_t const& InputSize() const { return inSize; } + size_t InputSize() const { return inSize; } //! Get the output size. - size_t const& OutputSize() const { return outSize; } + size_t OutputSize() const { return outSize; } //! Get the kernel width. - size_t const& KernelWidth() const { return kernelWidth; } + size_t KernelWidth() const { return kernelWidth; } //! Modify the kernel width. size_t& KernelWidth() { return kernelWidth; } //! Get the kernel height. - size_t const& KernelHeight() const { return kernelHeight; } + size_t KernelHeight() const { return kernelHeight; } //! Modify the kernel height. size_t& KernelHeight() { return kernelHeight; } //! Get the stride width. - size_t const& StrideWidth() const { return strideWidth; } + size_t StrideWidth() const { return strideWidth; } //! Modify the stride width. size_t& StrideWidth() { return strideWidth; } //! Get the stride height. - size_t const& StrideHeight() const { return strideHeight; } + size_t StrideHeight() const { return strideHeight; } //! Modify the stride height. size_t& StrideHeight() { return strideHeight; } //! Get the top padding height. - size_t const& PadHTop() const { return padHTop; } + size_t PadHTop() const { return padHTop; } //! Modify the top padding height. size_t& PadHTop() { return padHTop; } //! Get the bottom padding height. - size_t const& PadHBottom() const { return padHBottom; } + size_t PadHBottom() const { return padHBottom; } //! Modify the bottom padding height. size_t& PadHBottom() { return padHBottom; } //! Get the left padding width. - size_t const& PadWLeft() const { return padWLeft; } + size_t PadWLeft() const { return padWLeft; } //! Modify the left padding width. size_t& PadWLeft() { return padWLeft; } //! Get the right padding width. - size_t const& PadWRight() const { return padWRight; } + size_t PadWRight() const { return padWRight; } //! Modify the right padding width. size_t& PadWRight() { return padWRight; } + //! Get the shape of the input. + size_t InputShape() const + { + return inputHeight * inputWidth * inSize; + } + //! Get the size of the weight matrix. size_t WeightSize() const { return (outSize * inSize * kernelWidth * kernelHeight) + outSize; } - - const std::vector& OutputDimensions() const - { - std::vector result(inputDimensions.size(), 0); - result[0] = outputWidth; - result[1] = outputHeight; - // Higher dimensions are unmodified. - for (size_t i = 2; i < inputDimensions.size(); ++i) - result[i] = inputDimensions[i]; - return result; - } - /** * Serialize the layer. */ @@ -413,13 +425,13 @@ class TransposedConvolutionType : public Layer size_t aH; //! Locally-stored weight object. - OutputType weights; + OutputDataType weights; //! Locally-stored weight object. - arma::Cube weight; + arma::cube weight; //! Locally-stored bias term object. - OutputType bias; + arma::mat bias; //! Locally-stored input width. size_t inputWidth; @@ -434,35 +446,38 @@ class TransposedConvolutionType : public Layer size_t outputHeight; //! Locally-stored transformed output parameter. - arma::Cube outputTemp; + arma::cube outputTemp; //! Locally-stored transformed padded input parameter. - arma::Cube inputPaddedTemp; + arma::cube inputPaddedTemp; //! Locally-stored transformed expanded input parameter. - arma::Cube inputExpandedTemp; + arma::cube inputExpandedTemp; //! Locally-stored transformed error parameter. - arma::Cube gTemp; + arma::cube gTemp; //! Locally-stored transformed gradient parameter. - arma::Cube gradientTemp; + arma::cube gradientTemp; //! Locally-stored padding layer for forward propagation. - ann::Padding paddingForward; + ann::Padding<> paddingForward; //! Locally-stored padding layer for back propagation. - ann::Padding paddingBackward; -}; // class TransposedConvolutionType + ann::Padding<> paddingBackward; -// Standard TransposedConvolution -typedef TransposedConvolutionType< - NaiveConvolution, - NaiveConvolution, - NaiveConvolution, - arma::mat, - arma::mat -> TransposedConvolution; + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class TransposedConvolution } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/transposed_convolution_impl.hpp b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp similarity index 79% rename from src/mlpack/methods/ann/layer/not_adapted/transposed_convolution_impl.hpp rename to src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp index a660133fe9..d932acb6a7 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/transposed_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp @@ -23,16 +23,16 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > -TransposedConvolutionType< +TransposedConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType ->::TransposedConvolutionType() + InputDataType, + OutputDataType +>::TransposedConvolution() { // Nothing to do here. } @@ -41,16 +41,16 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > -TransposedConvolutionType< +TransposedConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType ->::TransposedConvolutionType( + InputDataType, + OutputDataType +>::TransposedConvolution( const size_t inSize, const size_t outSize, const size_t kernelWidth, @@ -64,7 +64,7 @@ TransposedConvolutionType< const size_t outputWidth, const size_t outputHeight, const std::string& paddingType) : - TransposedConvolutionType( + TransposedConvolution( inSize, outSize, kernelWidth, @@ -86,16 +86,16 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > -TransposedConvolutionType< +TransposedConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType ->::TransposedConvolutionType( + InputDataType, + OutputDataType +>::TransposedConvolution( const size_t inSize, const size_t outSize, const size_t kernelWidth, @@ -152,10 +152,10 @@ TransposedConvolutionType< const size_t padWidthRightForward = kernelWidth - padWRight - 1; const size_t padHeightBottomtForward = kernelHeight - padHBottom - 1; - paddingForward = ann::Padding(padWidthLeftForward, + paddingForward = ann::Padding<>(padWidthLeftForward, padWidthRightForward + aW, padHeightTopForward, padHeightBottomtForward + aH); - paddingBackward = ann::Padding(padWLeft, padWRight, padHTop, padHBottom); + paddingBackward = ann::Padding<>(padWLeft, padWRight, padHTop, padHBottom); // Check if the output height and width are possible given the other // parameters of the layer. @@ -174,42 +174,42 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > -void TransposedConvolutionType< +void TransposedConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType ->::SetWeights(typename OutputType::elem_type* weightPtr) + InputDataType, + OutputDataType +>::Reset() { - weight = arma::Cube(weightPtr, - kernelWidth, kernelHeight, outSize * inSize, false, false); - bias = arma::Mat(weightsPtr + - weight.n_elem, outSize, 1, false, false); + weight = arma::cube(weights.memptr(), kernelWidth, kernelHeight, + outSize * inSize, false, false); + bias = arma::mat(weights.memptr() + weight.n_elem, + outSize, 1, false, false); } template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > -void TransposedConvolutionType< +template +void TransposedConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType ->::Forward(const InputType& input, OutputType& output) + InputDataType, + OutputDataType +>::Forward(const arma::Mat& input, arma::Mat& output) { batchSize = input.n_cols; - arma::Cube inputTemp( - const_cast(input).memptr(), inputWidth, inputHeight, - inSize * batchSize, false, false); + arma::cube inputTemp(const_cast&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); if (strideWidth > 1 || strideHeight > 1) { @@ -231,9 +231,9 @@ void TransposedConvolutionType< } else { - inputPaddedTemp = arma::Cube( - inputExpandedTemp.memptr(), inputExpandedTemp.n_rows, - inputExpandedTemp.n_cols, inputExpandedTemp.n_slices, false, false);; + inputPaddedTemp = arma::Cube(inputExpandedTemp.memptr(), + inputExpandedTemp.n_rows, inputExpandedTemp.n_cols, + inputExpandedTemp.n_slices, false, false);; } } else if (paddingForward.PadWLeft() != 0 || @@ -253,8 +253,8 @@ void TransposedConvolutionType< } output.set_size(outputWidth * outputHeight * outSize, batchSize); - outputTemp = arma::Cube(output.memptr(), - outputWidth, outputHeight, outSize * batchSize, false, false); + outputTemp = arma::Cube(output.memptr(), outputWidth, outputHeight, + outSize * batchSize, false, false); outputTemp.zeros(); for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < @@ -268,7 +268,7 @@ void TransposedConvolutionType< for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { - OutputType convOutput, rotatedFilter; + arma::Mat convOutput, rotatedFilter; Rotate180(weight.slice(outMapIdx), rotatedFilter); if (strideWidth > 1 || @@ -298,22 +298,22 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > -void TransposedConvolutionType< +template +void TransposedConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::Backward( - const InputType& /* input */, const OutputType& gy, OutputType& g) + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { - arma::Cube mappedError( - ((OutputType&) gy).memptr(), outputWidth, + arma::Cube mappedError(((arma::Mat&) gy).memptr(), outputWidth, outputHeight, outSize * batchSize, false, false); - arma::Cube mappedErrorPadded; + arma::Cube mappedErrorPadded; if (paddingBackward.PadWLeft() != 0 || paddingBackward.PadWRight() != 0 || paddingBackward.PadHTop() != 0 || paddingBackward.PadHBottom() != 0) { @@ -329,8 +329,8 @@ void TransposedConvolutionType< } } g.set_size(inputWidth * inputHeight * inSize, batchSize); - gTemp = arma::Cube(g.memptr(), inputWidth, - inputHeight, inSize * batchSize, false, false); + gTemp = arma::Cube(g.memptr(), inputWidth, inputHeight, inSize * + batchSize, false, false); gTemp.zeros(); @@ -345,7 +345,7 @@ void TransposedConvolutionType< for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { - OutputType output; + arma::Mat output; if (paddingBackward.PadWLeft() != 0 || paddingBackward.PadWRight() != 0 || paddingBackward.PadHTop() != 0 || paddingBackward.PadHBottom() != 0) @@ -368,33 +368,32 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > -void TransposedConvolutionType< +template +void TransposedConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::Gradient( - const InputType& input, - const OutputType& error, - OutputType& gradient) + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) { - arma::Cube mappedError( - ((OutputType&) error).memptr(), outputWidth, outputHeight, - outSize * batchSize, false, false); - arma::Cube inputTemp( - const_cast(input).memptr(), inputWidth, inputHeight, - inSize * batchSize, false, false); + arma::Cube mappedError(((arma::Mat&) error).memptr(), outputWidth, + outputHeight, outSize * batchSize, false, false); + arma::cube inputTemp(const_cast&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); gradient.set_size(weights.n_elem, 1); - gradientTemp = arma::Cube(gradient.memptr(), - weight.n_rows, weight.n_cols, weight.n_slices, false, false); + gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, + weight.n_cols, weight.n_slices, false, false); gradientTemp.zeros(); - OutputType inputSlice, output, deltaSlice, rotatedOutput; + arma::Mat inputSlice, output, deltaSlice, rotatedOutput; for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < outSize * batchSize; outMap++) @@ -438,20 +437,18 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > template -void TransposedConvolutionType< +void TransposedConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::serialize(Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(inSize)); ar(CEREAL_NVP(outSize)); ar(CEREAL_NVP(batchSize)); @@ -472,6 +469,8 @@ void TransposedConvolutionType< if (cereal::is_loading()) { + weights.set_size((outSize * inSize * kernelWidth * kernelHeight) + outSize, + 1); size_t totalPadWidth = padWLeft + padWRight; size_t totalPadHeight = padHTop + padHBottom; aW = (outputWidth + kernelWidth - totalPadWidth - 2) % strideWidth; @@ -483,20 +482,20 @@ template< typename ForwardConvolutionRule, typename BackwardConvolutionRule, typename GradientConvolutionRule, - typename InputType, - typename OutputType + typename InputDataType, + typename OutputDataType > -void TransposedConvolutionType< +void TransposedConvolution< ForwardConvolutionRule, BackwardConvolutionRule, GradientConvolutionRule, - InputType, - OutputType + InputDataType, + OutputDataType >::InitializeSamePadding(){ /** * Using O=s*(I-1) + K -2P + A * where - * s=stride + * s=stride * I=Input Shape * K=Kernel Size * P=Padding diff --git a/src/mlpack/methods/ann/layer/not_adapted/virtual_batch_norm.hpp b/src/mlpack/methods/ann/layer/virtual_batch_norm.hpp similarity index 60% rename from src/mlpack/methods/ann/layer/not_adapted/virtual_batch_norm.hpp rename to src/mlpack/methods/ann/layer/virtual_batch_norm.hpp index 1d47a44e54..43b9a1d3a3 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/virtual_batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/virtual_batch_norm.hpp @@ -13,16 +13,14 @@ #define MLPACK_METHODS_ANN_LAYER_VIRTUALBATCHNORM_HPP #include -#include "layer.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { -// TODO: what about sizes for this layer? /** * Declaration of the VirtualBatchNorm layer class. Instead of using the - * batch statistics for normalizing on a mini-batch, it uses a reference subset - * of the data for calculating the normalization statistics. + * batch statistics for normalizing on a mini-batch, it uses a reference subset of + * the data for calculating the normalization statistics. * * For more information, refer to the following paper, * @@ -36,55 +34,49 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam InputType Type of the input data (arma::colvec, arma::mat, + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). - * @tparam OutputType Type of the output data (arma::colvec, arma::mat, + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat > -class VirtualBatchNormType : public Layer +class VirtualBatchNorm { public: //! Create the VirtualBatchNorm object. - VirtualBatchNormType(); + VirtualBatchNorm(); /** - * Create the VirtualBatchNorm layer object for a specified number of input - * units. + * Create the VirtualBatchNorm layer object for a specified number of input units. * * @param referenceBatch The data from which the normalization * statistics are computed. * @param size The number of input units / channels. * @param eps The epsilon added to variance to ensure numerical stability. */ - VirtualBatchNormType(const InputType& referenceBatch, - const size_t size, - const double eps = 1e-8); - - //! Clone the VirtualBatchNormType object. This handles polymorphism - //! correctly. - VirtualBatchNormType* Clone() const - { - return new VirtualBatchNormType(*this); - } + template + VirtualBatchNorm(const arma::Mat& referenceBatch, + const size_t size, + const double eps = 1e-8); /** * Reset the layer parameters. */ - void SetWeights(typename OutputType::elem_type* weightsPtr); + void Reset(); /** - * Forward pass of the Virtual Batch Normalization layer. Transforms the input - * data into zero mean and unit variance, scales the data by a factor gamma - * and shifts it by beta. + * Forward pass of the Virtual Batch Normalization layer. Transforms the input data + * into zero mean and unit variance, scales the data by a factor gamma and + * shifts it by beta. * * @param input Input data for the layer. * @param output Resulting output activations. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Backward pass through the layer. @@ -93,9 +85,10 @@ class VirtualBatchNormType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& /* input */, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& /* input */, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta and the input activations. @@ -104,14 +97,30 @@ class VirtualBatchNormType : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& /* input */, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient); //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } //! Get the number of input units. size_t InSize() const { return size; } @@ -119,11 +128,6 @@ class VirtualBatchNormType : public Layer //! Get the epsilon value. double Epsilon() const { return eps; } - const size_t WeightSize() const - { - return 2 * size; - } - /** * Serialize the layer. */ @@ -141,19 +145,19 @@ class VirtualBatchNormType : public Layer bool loading; //! Locally-stored scale parameter. - OutputType gamma; + OutputDataType gamma; //! Locally-stored shift parameter. - OutputType beta; + OutputDataType beta; //! Locally-stored parameters. - OutputType weights; + OutputDataType weights; //! Mean of features in the reference batch. - OutputType referenceBatchMean; + OutputDataType referenceBatchMean; //! Variance of features in the reference batch. - OutputType referenceBatchMeanSquared; + OutputDataType referenceBatchMeanSquared; //! The coefficient for reference batch statistics. double oldCoefficient; @@ -162,20 +166,29 @@ class VirtualBatchNormType : public Layer double newCoefficient; //! Locally-stored mean object. - OutputType mean; + OutputDataType mean; //! Locally-stored variance object. - OutputType variance; + OutputDataType variance; + + //! Locally-stored gradient object. + OutputDataType gradient; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored input parameter object. + OutputDataType inputParameter; //! Locally-stored normalized input. - OutputType normalized; + OutputDataType normalized; //! Locally-stored zero mean input. - OutputType inputSubMean; -}; // class VirtualBatchNormType - -// Standard VirtualBatchNorm layer. -typedef VirtualBatchNormType VirtualBatchNorm; + OutputDataType inputSubMean; +}; // class VirtualBatchNorm } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/virtual_batch_norm_impl.hpp b/src/mlpack/methods/ann/layer/virtual_batch_norm_impl.hpp similarity index 61% rename from src/mlpack/methods/ann/layer/not_adapted/virtual_batch_norm_impl.hpp rename to src/mlpack/methods/ann/layer/virtual_batch_norm_impl.hpp index c87d5e6579..112c625b9b 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/virtual_batch_norm_impl.hpp +++ b/src/mlpack/methods/ann/layer/virtual_batch_norm_impl.hpp @@ -19,8 +19,8 @@ namespace mlpack { namespace ann { /** Artificial Neural Network. */ -template -VirtualBatchNormType::VirtualBatchNormType() : +template +VirtualBatchNorm::VirtualBatchNorm() : size(0), eps(1e-8), loading(false), @@ -29,27 +29,29 @@ VirtualBatchNormType::VirtualBatchNormType() : { // Nothing to do here. } -template -VirtualBatchNormType::VirtualBatchNormType( - const InputType& referenceBatch, +template +template +VirtualBatchNorm::VirtualBatchNorm( + const arma::Mat& referenceBatch, const size_t size, const double eps) : size(size), eps(eps), loading(false) { + weights.set_size(size + size, 1); + referenceBatchMean = arma::mean(referenceBatch, 1); referenceBatchMeanSquared = arma::mean(arma::square(referenceBatch), 1); newCoefficient = 1.0 / (referenceBatch.n_cols + 1); oldCoefficient = 1 - newCoefficient; } -template -void VirtualBatchNormType::SetWeights( - typename OutputType::elem_type* weightsPtr) +template +void VirtualBatchNorm::Reset() { - gamma = OutputType(weightsPtr, size, 1, false, false); - beta = OutputType(weightsPtr + gamma.n_elem, size, 1, false, false); + gamma = arma::mat(weights.memptr(), size, 1, false, false); + beta = arma::mat(weights.memptr() + gamma.n_elem, size, 1, false, false); if (!loading) { @@ -60,19 +62,20 @@ void VirtualBatchNormType::SetWeights( loading = false; } -template -void VirtualBatchNormType::Forward( - const InputType& input, OutputType& output) +template +template +void VirtualBatchNorm::Forward( + const arma::Mat& input, arma::Mat& output) { Log::Assert(input.n_rows % size == 0, "Input features must be divisible \ by feature maps."); inputParameter = input; - InputType inputMean = arma::mean(input, 1); - InputType inputMeanSquared = arma::mean(arma::square(input), 1); + arma::mat inputMean = arma::mean(input, 1); + arma::mat inputMeanSquared = arma::mean(arma::square(input), 1); mean = oldCoefficient * referenceBatchMean + newCoefficient * inputMean; - OutputType meanSquared = oldCoefficient * referenceBatchMeanSquared + + arma::mat meanSquared = oldCoefficient * referenceBatchMeanSquared + newCoefficient * inputMeanSquared; variance = meanSquared - arma::square(mean); // Normalize the input. @@ -87,19 +90,18 @@ void VirtualBatchNormType::Forward( output.each_col() += beta; } -template -void VirtualBatchNormType::Backward( - const InputType& /* input */, - const OutputType& gy, - OutputType& g) +template +template +void VirtualBatchNorm::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) { - const OutputType stdInv = 1.0 / arma::sqrt(variance + eps); + const arma::mat stdInv = 1.0 / arma::sqrt(variance + eps); // dl / dxhat. - const OutputType norm = gy.each_col() % gamma; + const arma::mat norm = gy.each_col() % gamma; // sum dl / dxhat * (x - mu) * -0.5 * stdInv^3. - const OutputType var = arma::sum(norm % inputSubMean, 1) % + const arma::mat var = arma::sum(norm % inputSubMean, 1) % arma::pow(stdInv, 3.0) * -0.5; // dl / dxhat * 1 / stdInv + variance * 2 * (x - mu) / m + @@ -113,11 +115,12 @@ void VirtualBatchNormType::Backward( mean * -2)) * newCoefficient / inputParameter.n_cols; } -template -void VirtualBatchNormType::Gradient( - const InputType& /* input */, - const OutputType& error, - OutputType& gradient) +template +template +void VirtualBatchNorm::Gradient( + const arma::Mat& /* input */, + const arma::Mat& error, + arma::Mat& gradient) { gradient.set_size(size + size, 1); @@ -129,21 +132,22 @@ void VirtualBatchNormType::Gradient( arma::sum(error, 1); } -template +template template -void VirtualBatchNormType::serialize( +void VirtualBatchNorm::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); - ar(CEREAL_NVP(size)); - ar(CEREAL_NVP(eps)); if (cereal::is_loading()) { weights.set_size(size + size, 1); loading = true; } + + ar(CEREAL_NVP(eps)); + ar(CEREAL_NVP(gamma)); + ar(CEREAL_NVP(beta)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/vr_class_reward.hpp b/src/mlpack/methods/ann/layer/vr_class_reward.hpp similarity index 58% rename from src/mlpack/methods/ann/loss_functions/vr_class_reward.hpp rename to src/mlpack/methods/ann/layer/vr_class_reward.hpp index 8e0fe13a37..2d6f626c75 100644 --- a/src/mlpack/methods/ann/loss_functions/vr_class_reward.hpp +++ b/src/mlpack/methods/ann/layer/vr_class_reward.hpp @@ -1,8 +1,8 @@ /** - * @file methods/ann/loss_functions/vr_class_reward.hpp + * @file methods/ann/layer/vr_class_reward.hpp * @author Marcus Edel * - * Definition of the VRClassRewardType class, which implements the variance + * Definition of the VRClassReward class, which implements the variance * reduced classification reinforcement layer. * * mlpack is free software; you may redistribute it and/or modify it under the @@ -10,34 +10,41 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTIONS_VR_CLASS_REWARD_HPP -#define MLPACK_METHODS_ANN_LOSS_FUNCTIONS_VR_CLASS_REWARD_HPP +#ifndef MLPACK_METHODS_ANN_LAYER_VR_CLASS_REWARD_HPP +#define MLPACK_METHODS_ANN_LAYER_VR_CLASS_REWARD_HPP #include +#include "layer_types.hpp" + namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** * Implementation of the variance reduced classification reinforcement layer. * This layer is meant to be used in combination with the reinforce normal layer - * (ReinforceNormalLayer), which expects that the reward is 1 for success, and 0 - * otherwise. + * (ReinforceNormalLayer), which expects that an reward: + * (1 for success, 0 otherwise). * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class VRClassRewardType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class VRClassReward { public: /** - * Create the VRClassRewardType object. + * Create the VRClassReward object. * * @param scale Parameter used to scale the reward. * @param sizeAverage Take the average over all batches. */ - VRClassRewardType(const double scale = 1, const bool sizeAverage = true); + VRClassReward(const double scale = 1, const bool sizeAverage = true); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -47,8 +54,8 @@ class VRClassRewardType * @param target The target vector, that contains the class index in the range * between 1 and the number of classes. */ - typename MatType::elem_type Forward(const MatType& input, - const MatType& target); + template + double Forward(const InputType& input, const TargetType& target); /** * Ordinary feed backward pass of a neural network. The negative log @@ -61,30 +68,38 @@ class VRClassRewardType * between 1 and the number of classes. * @param output The calculated error. */ - void Backward(const MatType& input, const MatType& target, MatType& output); + template + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); - /** + //! Get the output parameter. + OutputDataType& OutputParameter() const {return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const {return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + /* * Add a new module to the model. * * @param args The layer parameter. */ - template + template void Add(Args... args) { network.push_back(new LayerType(args...)); } - /** + /* * Add a new module to the model. * * @param layer The Layer to be added to the model. */ - void Add(Layer* layer) - { - network.push_back(layer); - } + void Add(LayerTypes<> layer) { network.push_back(layer); } - //! Get the network. - const std::vector*>& Network() const { return network; } - //! Modify the network. - std::vector*>& Network() { return network; } + //! Get the network modules. + std::vector >& Model() { return network; } //! Get the value of parameter sizeAverage. bool SizeAverage() const { return sizeAverage; } @@ -108,12 +123,15 @@ class VRClassRewardType //! Locally stored reward parameter. double reward; - //! Locally-stored network modules. - std::vector*> network; -}; // class VRClassRewardType + //! Locally-stored delta object. + OutputDataType delta; -// Default typedef for typical `arma::mat` usage. -typedef VRClassRewardType VRClassReward; + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored network modules. + std::vector > network; +}; // class VRClassReward } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/vr_class_reward_impl.hpp b/src/mlpack/methods/ann/layer/vr_class_reward_impl.hpp new file mode 100644 index 0000000000..c04da10594 --- /dev/null +++ b/src/mlpack/methods/ann/layer/vr_class_reward_impl.hpp @@ -0,0 +1,107 @@ +/** + * @file methods/ann/layer/vr_class_reward_impl.hpp + * @author Marcus Edel + * + * Implementation of the VRClassReward class, which implements the variance + * reduced classification reinforcement layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_VR_CLASS_REWARD_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_VR_CLASS_REWARD_IMPL_HPP + +// In case it hasn't yet been included. +#include "vr_class_reward.hpp" + +#include "../visitor/reward_set_visitor.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +VRClassReward::VRClassReward( + const double scale, + const bool sizeAverage) : + scale(scale), + sizeAverage(sizeAverage), + reward(0) +{ + // Nothing to do here. +} + +template +template +double VRClassReward::Forward( + const InputType& input, const TargetType& target) +{ + double output = 0; + for (size_t i = 0; i < input.n_cols - 1; ++i) + { + size_t currentTarget = target(i) - 1; + Log::Assert(currentTarget < input.n_rows, + "Target class out of range."); + + output -= input(currentTarget, i); + } + + reward = 0; + arma::uword index = 0; + + for (size_t i = 0; i < input.n_cols - 1; ++i) + { + input.unsafe_col(i).max(index); + reward = ((index + 1) == target(i)) * scale; + } + + if (sizeAverage) + { + return output - reward / (input.n_cols - 1); + } + + return output - reward; +} + +template +template +void VRClassReward::Backward( + const InputType& input, + const TargetType& target, + OutputType& output) +{ + output = arma::zeros(input.n_rows, input.n_cols); + for (size_t i = 0; i < (input.n_cols - 1); ++i) + { + size_t currentTarget = target(i) - 1; + Log::Assert(currentTarget < input.n_rows, + "Target class out of range."); + + output(currentTarget, i) = -1; + } + + double vrReward = reward - input(0, 1); + if (sizeAverage) + { + vrReward /= input.n_cols - 1; + } + + const double norm = sizeAverage ? 2.0 / (input.n_cols - 1) : 2.0; + + output(0, 1) = norm * (input(0, 1) - reward); + boost::apply_visitor(RewardSetVisitor(vrReward), network.back()); +} + +template +template +void VRClassReward::serialize( + Archive& /* ar */, const uint32_t /* version */) +{ + // Nothing to do here. +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/not_adapted/weight_norm.hpp b/src/mlpack/methods/ann/layer/weight_norm.hpp similarity index 58% rename from src/mlpack/methods/ann/layer/not_adapted/weight_norm.hpp rename to src/mlpack/methods/ann/layer/weight_norm.hpp index b8e39326b0..2bdcd7e955 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/weight_norm.hpp +++ b/src/mlpack/methods/ann/layer/weight_norm.hpp @@ -13,7 +13,14 @@ #define MLPACK_METHODS_ANN_LAYER_WEIGHTNORM_HPP #include -#include "layer.hpp" +#include "layer_types.hpp" + +#include "../visitor/delete_visitor.hpp" +#include "../visitor/delta_visitor.hpp" +#include "../visitor/output_parameter_visitor.hpp" +#include "../visitor/reset_visitor.hpp" +#include "../visitor/weight_size_visitor.hpp" +#include "../visitor/weight_set_visitor.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -44,45 +51,30 @@ namespace ann /** Artificial Neural Network. */ { * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). + * @tparam CustomLayers Additional custom layers that can be added. */ template < - typename InputType = arma::mat, - typename OutputType = arma::mat + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename... CustomLayers > -class WeightNormType : public Layer +class WeightNorm { public: - /** - * Create an empty WeightNorm layer. - */ - WeightNormType(); - /** * Create the WeightNorm layer object. * * @param layer The layer whose weights are needed to be normalized. */ - WeightNormType(Layer* layer); + WeightNorm(LayerTypes layer = LayerTypes()); //! Destructor to release allocated memory. - ~WeightNormType(); - - //! Create a WeightNorm layer by copying the given layer. - WeightNormType(const WeightNormType& other); - //! Create a WeightNorm layer by taking ownership of the other layer. - WeightNormType(WeightNormType&& other); - //! Copy the given layer. - WeightNormType& operator=(const WeightNormType& other); - //! Take ownership of the data in the given layer. - WeightNormType& operator=(WeightNormType&& other); - - //! Clone the WeightNormType object. This handles polymorphism correctly. - WeightNormType* Clone() const { return new WeightNormType(*this); } + ~WeightNorm(); /** * Reset the layer parameters. */ - void SetWeights(typename OutputType::elem_type* weightsPtr); + void Reset(); /** * Forward pass of the WeightNorm layer. Calculates the weights of the @@ -93,7 +85,8 @@ class WeightNormType : public Layer * @param input Input data for the layer. * @param output Resulting output activations. */ - void Forward(const InputType& input, OutputType& output); + template + void Forward(const arma::Mat& input, arma::Mat& output); /** * Backward pass through the layer. This function calls the Backward() @@ -103,9 +96,10 @@ class WeightNormType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, - const OutputType& gy, - OutputType& g); + template + void Backward(const arma::Mat& input, + const arma::Mat& gy, + arma::Mat& g); /** * Calculate the gradient using the output delta, input activations and the @@ -115,25 +109,33 @@ class WeightNormType : public Layer * @param error The calculated error. * @param gradient The calculated gradient. */ - void Gradient(const InputType& input, - const OutputType& error, - OutputType& gradient); + template + void Gradient(const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient); + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the parameters. - OutputType const& Parameters() const { return weights; } + OutputDataType const& Parameters() const { return weights; } //! Modify the parameters. - OutputType& Parameters() { return weights; } + OutputDataType& Parameters() { return weights; } //! Get the wrapped layer. - Layer* const& WrappedLayer() { return wrappedLayer; } - - const size_t WeightSize() const { return wrappedLayer->WeightSize(); } - - const std::vector OutputDimensions() const - { - wrappedLayer->InputDimensions() = inputDimensions; - return wrappedLayer->OutputDimensions(); - } + LayerTypes const& Layer() { return wrappedLayer; } /** * Serialize the layer. @@ -145,33 +147,54 @@ class WeightNormType : public Layer //! Locally-stored number of bias elements in the weights of wrapped layer. size_t biasWeightSize; + //! Locally-stored delete visitor module object. + DeleteVisitor deleteVisitor; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored delta visitor module object. + DeltaVisitor deltaVisitor; + + //! Locally-stored gradient object. + OutputDataType gradient; + //! Locally-stored wrapped layer. - Layer* wrappedLayer; + LayerTypes wrappedLayer; //! Locally stored number of elements in the weights of wrapped layer. size_t layerWeightSize; + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored output parameter visitor module object. + OutputParameterVisitor outputParameterVisitor; + //! Reset the gradient for all modules that implement the Gradient function. - void ResetGradients(OutputType& gradient); + void ResetGradients(arma::mat& gradient); + + //! Locally-stored reset visitor. + ResetVisitor resetVisitor; //! Locally-stored scalar parameter. - OutputType scalarParameter; + OutputDataType scalarParameter; //! Locally-stored parameter vector. - OutputType vectorParameter; + OutputDataType vectorParameter; //! Locally-stored parameters. - OutputType weights; + OutputDataType weights; + + //! Locally-stored weight size visitor. + WeightSizeVisitor weightSizeVisitor; //! Locally-stored gradients of wrappedLayer. - OutputType layerGradients; + OutputDataType layerGradients; //! Locally-stored weights of wrappedLayer. - OutputType layerWeights; -}; // class WeightNormType. - -// Standard WeightNorm layer. -typedef WeightNormType WeightNorm; + OutputDataType layerWeights; +}; // class WeightNorm } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/weight_norm_impl.hpp b/src/mlpack/methods/ann/layer/weight_norm_impl.hpp new file mode 100644 index 0000000000..c534cc36df --- /dev/null +++ b/src/mlpack/methods/ann/layer/weight_norm_impl.hpp @@ -0,0 +1,165 @@ +/** + * @file methods/ann/layer/weight_norm_impl.hpp + * @author Toshal Agrawal + * + * Implementation of the WeightNorm Layer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_LAYER_WEIGHTNORM_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_WEIGHTNORM_IMPL_HPP + +// In case it is not included. +#include "weight_norm.hpp" + +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" +#include "../visitor/bias_set_visitor.hpp" + +namespace mlpack { +namespace ann { /** Artificial Neural Network. */ + +template +WeightNorm::WeightNorm( + LayerTypes layer) : + wrappedLayer(layer) +{ + layerWeightSize = boost::apply_visitor(weightSizeVisitor, wrappedLayer); + weights.set_size(layerWeightSize + 1, 1); + + layerWeights.set_size(layerWeightSize, 1); + layerGradients.set_size(layerWeightSize, 1); +} + +template +WeightNorm::~WeightNorm() +{ + boost::apply_visitor(deleteVisitor, wrappedLayer); +} + +template +void WeightNorm::Reset() +{ + // Set the weights of the inside layer to layerWeights. + // This is done to set the non-bias terms correctly. + boost::apply_visitor(WeightSetVisitor(layerWeights, 0), wrappedLayer); + + boost::apply_visitor(resetVisitor, wrappedLayer); + + biasWeightSize = boost::apply_visitor(BiasSetVisitor(weights, 0), + wrappedLayer); + + vectorParameter = arma::mat(weights.memptr() + biasWeightSize, + layerWeightSize - biasWeightSize, 1, false, false); + + scalarParameter = arma::mat(weights.memptr() + layerWeightSize, 1, 1, false, + false); +} + +template +template +void WeightNorm::Forward( + const arma::Mat& input, arma::Mat& output) +{ + // Initialize the non-bias weights of wrapped layer. + const double normVectorParameter = arma::norm(vectorParameter, 2); + layerWeights.rows(0, layerWeightSize - biasWeightSize - 1) = + scalarParameter(0) * vectorParameter / normVectorParameter; + + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, wrappedLayer)), + wrappedLayer); + + output = boost::apply_visitor(outputParameterVisitor, wrappedLayer); +} + +template +template +void WeightNorm::Backward( + const arma::Mat& /* input */, const arma::Mat& gy, arma::Mat& g) +{ + boost::apply_visitor(BackwardVisitor(boost::apply_visitor( + outputParameterVisitor, wrappedLayer), gy, + boost::apply_visitor(deltaVisitor, wrappedLayer)), wrappedLayer); + + g = boost::apply_visitor(deltaVisitor, wrappedLayer); +} + +template +template +void WeightNorm::Gradient( + const arma::Mat& input, + const arma::Mat& error, + arma::Mat& gradient) +{ + ResetGradients(layerGradients); + + // Calculate the gradients of the wrapped layer. + boost::apply_visitor(GradientVisitor(input, error), wrappedLayer); + + // Store the norm of vector parameter temporarily. + const double normVectorParameter = arma::norm(vectorParameter, 2); + + // Set the gradients of the bias terms. + if (biasWeightSize != 0) + { + gradient.rows(0, biasWeightSize - 1) = arma::mat(layerGradients.memptr() + + layerWeightSize - biasWeightSize, biasWeightSize, 1, false, false); + } + + // Calculate the gradients of the scalar parameter. + gradient[gradient.n_rows - 1] = arma::accu(layerGradients.rows(0, + layerWeightSize - biasWeightSize - 1) % vectorParameter) / + normVectorParameter; + + // Calculate the gradients of the vector parameter. + gradient.rows(biasWeightSize, layerWeightSize - 1) = + scalarParameter(0) / normVectorParameter * (layerGradients.rows(0, + layerWeightSize - biasWeightSize - 1) - gradient[gradient.n_rows - 1] / + normVectorParameter * vectorParameter); +} + +template +void WeightNorm::ResetGradients( + arma::mat& gradient) +{ + boost::apply_visitor(GradientSetVisitor(gradient, 0), wrappedLayer); +} + +template +template +void WeightNorm::serialize( + Archive& ar, const uint32_t /* version */) +{ + if (cereal::is_loading()) + { + boost::apply_visitor(deleteVisitor, wrappedLayer); + } + + ar(CEREAL_VARIANT_POINTER(wrappedLayer)); + ar(CEREAL_NVP(layerWeightSize)); + + // If we are loading, we need to initialize the weights. + if (cereal::is_loading()) + { + weights.set_size(layerWeightSize + 1, 1); + } +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index 35116a8831..a3c97ff4f6 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -47,8 +47,6 @@ set(SOURCES soft_margin_loss_impl.hpp triplet_margin_loss.hpp triplet_margin_loss_impl.hpp - vr_class_reward.hpp - vr_class_reward_impl.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp index eae005c1d6..8893227bc7 100644 --- a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp @@ -18,14 +18,19 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The binary-cross-entropy performance function measures the Binary Cross - * Entropy between the target and the output. + * The binary-cross-entropy performance function measures the + * Binary Cross Entropy between the target and the output. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class BCELossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class BCELoss { public: /** @@ -40,7 +45,7 @@ class BCELossType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - BCELossType(const double eps = 1e-10, const bool reduction = true); + BCELoss(const double eps = 1e-10, const bool reduction = true); /** * Computes the cross-entropy function. @@ -49,8 +54,9 @@ class BCELossType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -60,9 +66,15 @@ class BCELossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the epsilon. double Eps() const { return eps; } @@ -82,23 +94,25 @@ class BCELossType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! The minimum value used for computing logarithms and denominators double eps; //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class BCELossType - -// Default typedef for typical `arma::mat` usage. -typedef BCELossType BCELoss; +}; // class BCELoss /** - * Alias of BCELossType. + * Adding alias of BCELoss. */ -typedef BCELossType CrossEntropyError; - -template -using CrossEntropyErrorType = BCELossType; +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using CrossEntropyError = BCELoss< + InputDataType, OutputDataType>; } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp index 0855cc580c..a217141564 100644 --- a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp @@ -18,19 +18,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -BCELossType::BCELossType( +template +BCELoss::BCELoss( const double eps, const bool reduction) : eps(eps), reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type BCELossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +BCELoss::Forward( + const PredictionType& prediction, + const TargetType& target) { - typedef typename MatType::elem_type ElemType; + typedef typename PredictionType::elem_type ElemType; ElemType lossSum = -arma::accu(target % arma::log(prediction + eps) + (1. - target) % arma::log(1. - prediction + eps)); @@ -41,11 +43,12 @@ typename MatType::elem_type BCELossType::Forward( return lossSum / target.n_elem; } -template -void BCELossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void BCELoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss = (1. - target) / (1. - prediction + eps) - target / (prediction + eps); @@ -53,9 +56,9 @@ void BCELossType::Backward( loss /= target.n_elem; } -template +template template -void BCELossType::serialize( +void BCELoss::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp index 9fc2ee8d35..362d0d4e3b 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -26,16 +26,21 @@ namespace ann /** Artificial Neural Network. */ { * f(x) = 1 - cos(x1, x2) , for y = 1 * f(x) = max(0, cos(x1, x2) - margin) , for y = -1 * @f} - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class CosineEmbeddingLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class CosineEmbeddingLoss { public: /** - * Create the CosineEmbeddingLossType object. + * Create the CosineEmbeddingLoss object. * * @param margin Increases cosine distance in case of dissimilarity. * Refer definition of cosine-embedding-loss above. @@ -47,7 +52,7 @@ class CosineEmbeddingLossType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - CosineEmbeddingLossType(const double margin = 0.0, + CosineEmbeddingLoss(const double margin = 0.0, const bool similarity = true, const bool reduction = true); @@ -58,8 +63,9 @@ class CosineEmbeddingLossType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -69,9 +75,25 @@ class CosineEmbeddingLossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the input parameter. + InputDataType& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -96,6 +118,15 @@ class CosineEmbeddingLossType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Locally-stored value of margin hyper-parameter. double margin; @@ -104,10 +135,7 @@ class CosineEmbeddingLossType //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class CosineEmbeddingLossType - -// Default typedef for typical `arma::mat` usage. -typedef CosineEmbeddingLossType CosineEmbeddingLoss; +}; // class CosineEmbeddingLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp index b21c743653..bee9906032 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp @@ -18,28 +18,30 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -CosineEmbeddingLossType::CosineEmbeddingLossType( +template +CosineEmbeddingLoss::CosineEmbeddingLoss( const double margin, const bool similarity, const bool reduction): margin(margin), similarity(similarity), reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type CosineEmbeddingLossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +CosineEmbeddingLoss::Forward( + const PredictionType& prediction, + const TargetType& target) { - typedef typename MatType::elem_type ElemType; + typedef typename PredictionType::elem_type ElemType; const size_t cols = prediction.n_cols; const size_t batchSize = prediction.n_elem / cols; if (arma::size(prediction) != arma::size(target)) Log::Fatal << "Input Tensors must have same dimensions." << std::endl; - arma::Col inputTemp1 = arma::vectorise(prediction); - arma::Col inputTemp2 = arma::vectorise(target); + arma::colvec inputTemp1 = arma::vectorise(prediction); + arma::colvec inputTemp2 = arma::vectorise(target); ElemType lossSum = 0.0; for (size_t i = 0; i < inputTemp1.n_elem; i += cols) @@ -62,24 +64,25 @@ typename MatType::elem_type CosineEmbeddingLossType::Forward( return (ElemType) lossSum / batchSize; } -template -void CosineEmbeddingLossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void CosineEmbeddingLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { - typedef typename MatType::elem_type ElemType; + typedef typename PredictionType::elem_type ElemType; const size_t cols = prediction.n_cols; const size_t batchSize = prediction.n_elem / cols; if (arma::size(prediction) != arma::size(target)) Log::Fatal << "Input Tensors must have same dimensions." << std::endl; - arma::Col inputTemp1 = arma::vectorise(prediction); - arma::Col inputTemp2 = arma::vectorise(target); + arma::colvec inputTemp1 = arma::vectorise(prediction); + arma::colvec inputTemp2 = arma::vectorise(target); loss.set_size(arma::size(inputTemp1)); - arma::Col outputTemp(loss.memptr(), inputTemp1.n_elem, + arma::colvec outputTemp(loss.memptr(), inputTemp1.n_elem, false, false); for (size_t i = 0; i < inputTemp1.n_elem; i += cols) { @@ -103,9 +106,9 @@ void CosineEmbeddingLossType::Backward( } } -template +template template -void CosineEmbeddingLossType::serialize( +void CosineEmbeddingLoss::serialize( Archive& ar, const uint32_t /* version */) { ar(CEREAL_NVP(margin)); diff --git a/src/mlpack/methods/ann/loss_functions/dice_loss.hpp b/src/mlpack/methods/ann/loss_functions/dice_loss.hpp index 2e7b89b4be..ce4a80f54d 100644 --- a/src/mlpack/methods/ann/loss_functions/dice_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/dice_loss.hpp @@ -38,19 +38,24 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class DiceLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class DiceLoss { public: /** - * Create the DiceLossType object. + * Create the DiceLoss object. * * @param smooth The Laplace smoothing parameter. */ - DiceLossType(const double smooth = 1); + DiceLoss(const double smooth = 1); /** * Computes the dice loss function. @@ -59,8 +64,9 @@ class DiceLossType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -70,9 +76,15 @@ class DiceLossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the smooth. double Smooth() const { return smooth; } @@ -86,12 +98,12 @@ class DiceLossType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! The parameter to avoid overfitting. double smooth; -}; // class DiceLossType - -// Default typedef for typical `arma::mat` usage. -typedef DiceLossType DiceLoss; +}; // class DiceLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp index a98534d086..2a1835dc70 100644 --- a/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp @@ -18,38 +18,41 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -DiceLossType::DiceLossType(const double smooth) : smooth(smooth) +template +DiceLoss::DiceLoss( + const double smooth) : smooth(smooth) { // Nothing to do here. } -template -typename MatType::elem_type DiceLossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type DiceLoss + ::Forward(const PredictionType& prediction, + const TargetType& target) { return 1 - ((2 * arma::accu(target % prediction) + smooth) / - (arma::accu(target % target) + arma::accu( - prediction % prediction) + smooth)); + (arma::accu(target % target) + arma::accu( + prediction % prediction) + smooth)); } -template -void DiceLossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void DiceLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss = -2 * (target * (arma::accu(prediction % prediction) + - arma::accu(target % target) + smooth) - prediction * - (2 * arma::accu(target % prediction) + smooth)) / std::pow( - arma::accu(target % target) + arma::accu(prediction % prediction) - + smooth, 2.0); + arma::accu(target % target) + smooth) - prediction * + (2 * arma::accu(target % prediction) + smooth)) / std::pow( + arma::accu(target % target) + arma::accu(prediction % prediction) + + smooth, 2.0); } -template +template template -void DiceLossType::serialize( +void DiceLoss::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp index 4f6da12ea1..e8013aac57 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp @@ -21,15 +21,20 @@ namespace ann /** Artificial Neural Network. */ { * The earth mover distance function measures the network's performance * according to the Kantorovich-Rubinstein duality approximation. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class EarthMoverDistanceType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class EarthMoverDistance { public: /** - * Create the EarthMoverDistanceType object. + * Create the EarthMoverDistance object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -37,7 +42,7 @@ class EarthMoverDistanceType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - EarthMoverDistanceType(const bool reduction = true); + EarthMoverDistance(const bool reduction = true); /** * Ordinary feed forward pass of a neural network. @@ -46,8 +51,9 @@ class EarthMoverDistanceType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -57,9 +63,15 @@ class EarthMoverDistanceType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -74,12 +86,12 @@ class EarthMoverDistanceType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class EarthMoverDistanceType - -// Default typedef for typical `arma::mat` usage. -typedef EarthMoverDistanceType EarthMoverDistance; +}; // class EarthMoverDistance } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp index a0e0ecaeb1..a049e1dc5a 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp @@ -18,19 +18,22 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -EarthMoverDistanceType::EarthMoverDistanceType(const bool reduction) : - reduction(reduction) +template +EarthMoverDistance + ::EarthMoverDistance(const bool reduction) : reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type EarthMoverDistanceType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +EarthMoverDistance::Forward( + const PredictionType& prediction, + const TargetType& target) { - typename MatType::elem_type lossSum = -arma::accu(target % prediction); + typename PredictionType::elem_type lossSum = + -arma::accu(target % prediction); if (reduction) return lossSum; @@ -38,11 +41,12 @@ typename MatType::elem_type EarthMoverDistanceType::Forward( return lossSum / target.n_elem; } -template -void EarthMoverDistanceType::Backward( - const MatType& /* prediction */, - const MatType& target, - MatType& loss) +template +template +void EarthMoverDistance::Backward( + const PredictionType& /* prediction */, + const TargetType& target, + LossType& loss) { loss = -target; @@ -50,9 +54,9 @@ void EarthMoverDistanceType::Backward( loss = loss / target.n_elem; } -template +template template -void EarthMoverDistanceType::serialize( +void EarthMoverDistance::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/empty_loss.hpp b/src/mlpack/methods/ann/loss_functions/empty_loss.hpp index 525823a37f..8cc8caae9d 100644 --- a/src/mlpack/methods/ann/loss_functions/empty_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/empty_loss.hpp @@ -23,17 +23,22 @@ namespace ann /** Artificial Neural Network. */ { * The empty loss does nothing, letting the user calculate the loss outside * the model. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class EmptyLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class EmptyLoss { public: /** - * Create the EmptyLossType object. + * Create the EmptyLoss object. */ - EmptyLossType(); + EmptyLoss(); /** * Computes the Empty loss function. @@ -42,7 +47,8 @@ class EmptyLossType * function. * @param target The target vector. */ - double Forward(const MatType& input, const MatType& target); + template + double Forward(const PredictionType& input, const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -52,17 +58,11 @@ class EmptyLossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); - - //! Serialize the EmptyLossType. - template - void serialize(Archive& ar, const uint32_t /* version */) { } -}; // class EmptyLossType - -// Default typedef for typical `arma::mat` usage. -typedef EmptyLossType EmptyLoss; + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); +}; // class EmptyLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/empty_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/empty_loss_impl.hpp index 1083fbcfea..190792030e 100644 --- a/src/mlpack/methods/ann/loss_functions/empty_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/empty_loss_impl.hpp @@ -20,24 +20,26 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -EmptyLossType::EmptyLossType() +template +EmptyLoss::EmptyLoss() { // Nothing to do here. } -template -double EmptyLossType::Forward( - const MatType& /* prediction */, const MatType& /* target */) +template +template +double EmptyLoss::Forward( + const PredictionType& /* prediction */, const TargetType& /* target */) { return 0; } -template -void EmptyLossType::Backward( - const MatType& /* prediction */, - const MatType& target, - MatType& loss) +template +template +void EmptyLoss::Backward( + const PredictionType& /* prediction */, + const TargetType& target, + LossType& loss) { loss = target; } diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp index 40fe951650..74def487b8 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp @@ -24,11 +24,16 @@ namespace ann /** Artificial Neural Network. */ { * The Hinge Embedding loss function is often used to compute the loss * between y_true and y_pred. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class HingeEmbeddingLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class HingeEmbeddingLoss { public: /** @@ -40,7 +45,7 @@ class HingeEmbeddingLossType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - HingeEmbeddingLossType(const bool reduction = true); + HingeEmbeddingLoss(const bool reduction = true); /** * Computes the Hinge Embedding loss function. @@ -49,8 +54,9 @@ class HingeEmbeddingLossType * function. * @param target Target data to compare with. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -60,9 +66,15 @@ class HingeEmbeddingLossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -77,12 +89,13 @@ class HingeEmbeddingLossType void serialize(Archive& ar, const uint32_t /* version */); private: - //! Boolean values that tells if reduction is 'sum' or 'mean'. - bool reduction; -}; // class HingeEmbeddingLossType + //! Locally-stored output parameter object. + OutputDataType outputParameter; -// Default typedef for typical `arma::mat` usage. -typedef HingeEmbeddingLossType HingeEmbeddingLoss; + //! Boolean values that tells if reduction + // is 'sum' or 'mean'. + bool reduction; +}; // class HingeEmbeddingLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp index 26773b6523..701891d089 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp @@ -19,20 +19,22 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -HingeEmbeddingLossType::HingeEmbeddingLossType(const bool reduction) : - reduction(reduction) +template +HingeEmbeddingLoss + ::HingeEmbeddingLoss(const bool reduction) : reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type HingeEmbeddingLossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +HingeEmbeddingLoss::Forward( + const PredictionType& prediction, + const TargetType& target) { - MatType loss = (1 - target) / 2 + prediction % (target); - typename MatType::elem_type lossSum = arma::accu(loss); + PredictionType loss = (1 - target) / 2 + prediction % (target); + typename PredictionType::elem_type lossSum = arma::accu(loss); if (reduction) return lossSum; @@ -40,11 +42,12 @@ typename MatType::elem_type HingeEmbeddingLossType::Forward( return lossSum / target.n_elem; } -template -void HingeEmbeddingLossType::Backward( - const MatType& /* prediction */, - const MatType& target, - MatType& loss) +template +template +void HingeEmbeddingLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss = target; @@ -52,9 +55,9 @@ void HingeEmbeddingLossType::Backward( loss = loss / target.n_elem; } -template +template template -void HingeEmbeddingLossType::serialize( +void HingeEmbeddingLoss::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp index 7b1023078a..7829b20348 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp @@ -25,15 +25,20 @@ namespace ann /** Artificial Neural Network. */ { * The hinge loss \f$l(y_true, y_pred)\f$ is defined as * \f$l(y_true, y_pred) = max(0, 1 - y_true*y_pred)\f$. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class HingeLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class HingeLoss { public: /** - * Create HingeLossType object. + * Create HingeLoss object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -41,7 +46,7 @@ class HingeLossType * true, 'sum' reduction is used and the output will be * summed. It is set to true by default. */ - HingeLossType(const bool reduction = true); + HingeLoss(const bool reduction = true); /** * Computes the Hinge loss function. @@ -50,8 +55,9 @@ class HingeLossType * function. * @param target Target data to compare with. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -61,9 +67,15 @@ class HingeLossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -78,12 +90,12 @@ class HingeLossType void serialize(Archive& ar, const uint32_t /* version */); private: - //! The boolean value that tells if reduction is sum or mean. - bool reduction; -}; // class HingeLossType + //! Locally-stored output parameter object. + OutputDataType outputParameter; -// Default typedef for typical `arma::mat` usage. -typedef HingeLossType HingeLoss; + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; +}; // class HingeLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/hinge_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/hinge_loss_impl.hpp index 05403e32e8..6de5a553fa 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_loss_impl.hpp @@ -19,24 +19,26 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -HingeLossType::HingeLossType(const bool reduction): +template +HingeLoss::HingeLoss(const bool reduction): reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type HingeLossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +HingeLoss::Forward( + const PredictionType& prediction, + const TargetType& target) { - MatType temp = target - (target == 0); - MatType temp_zeros(size(target), arma::fill::zeros); + TargetType temp = target - (target == 0); + TargetType temp_zeros(size(target), arma::fill::zeros); - MatType loss = arma::max(temp_zeros, 1 - prediction % temp); + PredictionType loss = arma::max(temp_zeros, 1 - prediction % temp); - typename MatType::elem_type lossSum = arma::accu(loss); + typename PredictionType::elem_type lossSum = arma::accu(loss); if (reduction) return lossSum; @@ -44,22 +46,23 @@ typename MatType::elem_type HingeLossType::Forward( return lossSum / loss.n_elem; } -template -void HingeLossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void HingeLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { - MatType temp = target - (target == 0); + TargetType temp = target - (target == 0); loss = (prediction < (1 / temp)) % -temp; if (!reduction) loss /= target.n_elem; } -template +template template -void HingeLossType::serialize( +void HingeLoss::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp index 8596a260fc..c1533c84f6 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp @@ -24,15 +24,20 @@ namespace ann /** Artificial Neural Network. */ { * and linear for large values, with equal values and slopes of the different * sections at the two points where \f$ |y - f(x)| = delta \f$. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class HuberLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class HuberLoss { public: /** - * Create the HuberLossType object. + * Create the HuberLoss object. * * @param delta The threshold value upto which squared error is followed and * after which absolute error is considered. @@ -42,7 +47,7 @@ class HuberLossType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - HuberLossType(const double delta = 1.0, const bool reduction = true); + HuberLoss(const double delta = 1.0, const bool reduction = true); /** * Computes the Huber Loss function. @@ -51,8 +56,9 @@ class HuberLossType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -62,9 +68,15 @@ class HuberLossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the value of delta. double Delta() const { return delta; } @@ -84,15 +96,15 @@ class HuberLossType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Hyperparameter `delta` defines the point upto which MSE is considered. double delta; //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class HuberLossType - -// Default typedef for typical `arma::mat` usage. -typedef HuberLossType HuberLoss; +}; // class HuberLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp index 907a51e552..f2e496aaed 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp @@ -18,28 +18,30 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -HuberLossType::HuberLossType( - const double delta, - const bool reduction): - delta(delta), - reduction(reduction) +template +HuberLoss::HuberLoss( + const double delta, + const bool reduction): + delta(delta), + reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type HuberLossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +HuberLoss::Forward( + const PredictionType& prediction, + const TargetType& target) { - typedef typename MatType::elem_type ElemType; + typedef typename PredictionType::elem_type ElemType; ElemType lossSum = 0; for (size_t i = 0; i < prediction.n_elem; ++i) { - const ElemType absError = std::abs(target[i] - prediction[i]); - lossSum += absError > delta ? - delta * (absError - 0.5 * delta) : 0.5 * std::pow(absError, 2); + const ElemType absError = std::abs(target[i] - prediction[i]); + lossSum += absError > delta ? + delta * (absError - 0.5 * delta) : 0.5 * std::pow(absError, 2); } if (reduction) @@ -48,13 +50,14 @@ typename MatType::elem_type HuberLossType::Forward( return lossSum / target.n_elem; } -template -void HuberLossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void HuberLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { - typedef typename MatType::elem_type ElemType; + typedef typename PredictionType::elem_type ElemType; loss.set_size(size(prediction)); for (size_t i = 0; i < loss.n_elem; ++i) @@ -69,9 +72,9 @@ void HuberLossType::Backward( loss = loss / target.n_elem; } -template +template template -void HuberLossType::serialize( +void HuberLoss::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp index 16e6471da5..0962c3c4e4 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp @@ -33,11 +33,16 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class KLDivergenceType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class KLDivergence { public: /** @@ -50,7 +55,7 @@ class KLDivergenceType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - KLDivergenceType(const bool reduction = true); + KLDivergence(const bool reduction = true); /** * Computes the Kullback–Leibler divergence error function. @@ -59,8 +64,9 @@ class KLDivergenceType * function. * @param target Target data to compare with. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -70,29 +76,34 @@ class KLDivergenceType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } - /** - * Serialize the loss function. + * Serialize the loss function */ template void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class KLDivergenceType - -// Default typedef for typical `arma::mat` usage. -typedef KLDivergenceType KLDivergence; +}; // class KLDivergence } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp index 8088972b06..6ff54ac21b 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp @@ -19,20 +19,22 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -KLDivergenceType::KLDivergenceType(const bool reduction) : +template +KLDivergence::KLDivergence(const bool reduction): reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type KLDivergenceType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +KLDivergence::Forward( + const PredictionType& prediction, + const TargetType& target) { - MatType loss = target % (arma::log(target) - prediction); - typename MatType::elem_type lossSum = arma::accu(loss); + PredictionType loss = target % (arma::log(target) - prediction); + typename PredictionType::elem_type lossSum = arma::accu(loss); if (reduction) return lossSum; @@ -40,21 +42,22 @@ typename MatType::elem_type KLDivergenceType::Forward( return lossSum / target.n_elem; } -template -void KLDivergenceType::Backward( - const MatType& /* prediction */, - const MatType& target, - MatType& loss) +template +template +void KLDivergence::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { - loss = -target; + loss = - target; if (!reduction) loss = loss / target.n_elem; } -template +template template -void KLDivergenceType::serialize( +void KLDivergence::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp index cd02255fa9..e8494ca915 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp @@ -21,15 +21,20 @@ namespace ann /** Artificial Neural Network. */ { * The L1 loss is a loss function that measures the mean absolute error (MAE) * between each element in the input x and target y. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class L1LossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class L1Loss { public: /** - * Create the L1LossType object. + * Create the L1Loss object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -38,7 +43,7 @@ class L1LossType * is set to true by default. * */ - L1LossType(const bool reduction = true); + L1Loss(const bool reduction = true); /** * Computes the L1 Loss function. @@ -47,8 +52,9 @@ class L1LossType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -58,9 +64,15 @@ class L1LossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -75,12 +87,12 @@ class L1LossType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class L1LossType - -// Default typedef for typical `arma::mat` usage. -typedef L1LossType L1Loss; +}; // class L1Loss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp index 4694f40ba0..dfdfbc573d 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp @@ -18,20 +18,22 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -L1LossType::L1LossType(const bool reduction): - reduction(reduction) +template +L1Loss::L1Loss(const bool reduction): + reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type L1LossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +L1Loss::Forward( + const PredictionType& prediction, + const TargetType& target) { - MatType loss = arma::abs(prediction - target); - typename MatType::elem_type lossSum = arma::accu(loss); + PredictionType loss = arma::abs(prediction - target); + typename PredictionType::elem_type lossSum = arma::accu(loss); if (reduction) return lossSum; @@ -39,21 +41,22 @@ typename MatType::elem_type L1LossType::Forward( return lossSum / target.n_elem; } -template -void L1LossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void L1Loss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss = arma::sign(prediction - target); - + if (!reduction) loss = loss / target.n_elem; } -template +template template -void L1LossType::serialize( +void L1Loss::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp index 550f40279a..23aa6baa4e 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp @@ -23,11 +23,16 @@ namespace ann /** Artificial Neural Network. */ { * variational auto encoder. This function is the log of hyperbolic * cosine of difference between true values and predicted values. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class LogCoshLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class LogCoshLoss { public: /** @@ -45,7 +50,7 @@ class LogCoshLossType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - LogCoshLossType(const double a = 1.0, const bool reduction = true); + LogCoshLoss(const double a = 1.0, const bool reduction = true); /** * Computes the Log-Hyperbolic-Cosine loss function. @@ -54,8 +59,9 @@ class LogCoshLossType * function. * @param target Target data to compare with. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -65,9 +71,15 @@ class LogCoshLossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the value of hyperparameter a. double A() const { return a; } @@ -87,15 +99,15 @@ class LogCoshLossType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Hyperparameter a for smoothening function curve. double a; //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class LogCoshLossType - -// Default typedef for typical `arma::mat` usage. -typedef LogCoshLossType LogCoshLoss; +}; // class LogCoshLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp index 3ecfe59253..7c4bba7b9b 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp @@ -19,22 +19,22 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -LogCoshLossType::LogCoshLossType( - const double a, - const bool reduction) : - a(a), - reduction(reduction) +template +LogCoshLoss::LogCoshLoss( + const double a, const bool reduction) : + a(a) , reduction(reduction) { Log::Assert(a > 0, "Hyper-Parameter \'a\' must be positive"); } -template -typename MatType::elem_type LogCoshLossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +LogCoshLoss::Forward( + const PredictionType& prediction, + const TargetType& target) { - typename MatType::elem_type lossSum = + typename PredictionType::elem_type lossSum = arma::accu(arma::log(arma::cosh(a * (target - prediction)))) / a; if (reduction) @@ -43,11 +43,12 @@ typename MatType::elem_type LogCoshLossType::Forward( return lossSum / target.n_elem; } -template -void LogCoshLossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void LogCoshLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss = arma::tanh(a * (target - prediction)); @@ -55,9 +56,9 @@ void LogCoshLossType::Backward( loss = loss / target.n_elem; } -template +template template -void LogCoshLossType::serialize( +void LogCoshLoss::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index b158c444bc..fabb65d78a 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -22,19 +22,21 @@ namespace ann /** Artificial Neural Network. */ { * values of 1 or -1. If the label is 1 then the first input should be ranked * higher than the second input at a distance larger than a margin, and vice- * versa if the label is -1. - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class MarginRankingLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MarginRankingLoss { public: /** - * Create the MarginRankingLossType object with Hyperparameter margin. - * Hyperparameter margin defines a minimum distance between correctly ranked - * samples. - * + * Create the MarginRankingLoss object with Hyperparameter margin. * @param margin defines a minimum distance between correctly ranked samples. * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -42,7 +44,7 @@ class MarginRankingLossType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - MarginRankingLossType(const double margin = 1.0, const bool reduction = true); + MarginRankingLoss(const double margin = 1.0, const bool reduction = true); /** * Computes the Margin Ranking Loss function. @@ -51,8 +53,9 @@ class MarginRankingLossType * function. * @param target The label vector which contains values of -1 or 1. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -62,9 +65,19 @@ class MarginRankingLossType * @param target The label vector which contains -1 or 1 values. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template < + typename PredictionType, + typename TargetType, + typename LossType + > + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the margin parameter. double Margin() const { return margin; } @@ -84,15 +97,15 @@ class MarginRankingLossType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! The margin value used in calculating Margin Ranking Loss. double margin; //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class MarginRankingLossType - -// Default typedef for typical `arma::mat` usage. -typedef MarginRankingLossType MarginRankingLoss; +}; // class MarginRankingLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index 6551ac4df0..033357d9e8 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -18,24 +18,25 @@ namespace mlpack { namespace ann /** Artifical Neural Network. */ { -template -MarginRankingLossType::MarginRankingLossType( - const double margin, const bool reduction) : - margin(margin), - reduction(reduction) +template +MarginRankingLoss::MarginRankingLoss( + const double margin, const bool reduction): + margin(margin), reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type MarginRankingLossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +MarginRankingLoss::Forward( + const PredictionType& prediction, + const TargetType& target) { const int predictionRows = prediction.n_rows; - const MatType& prediction1 = prediction.rows(0, + const PredictionType& prediction1 = prediction.rows(0, predictionRows / 2 - 1); - const MatType& prediction2 = prediction.rows(predictionRows / 2, + const PredictionType& prediction2 = prediction.rows(predictionRows / 2, predictionRows - 1); double lossSum = arma::accu(arma::max(arma::zeros(size(target)), @@ -47,21 +48,26 @@ typename MatType::elem_type MarginRankingLossType::Forward( return lossSum / target.n_elem; } -template -void MarginRankingLossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template < + typename PredictionType, + typename TargetType, + typename LossType +> +void MarginRankingLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { const int predictionRows = prediction.n_rows; - const MatType& prediction1 = prediction.rows(0, + const PredictionType& prediction1 = prediction.rows(0, predictionRows / 2 - 1); - const MatType& prediction2 = prediction.rows(predictionRows / 2, + const PredictionType& prediction2 = prediction.rows(predictionRows / 2, predictionRows - 1); - MatType lossPrediction1 = -target % (prediction1 - prediction2) + margin; + LossType lossPrediction1 = -target % (prediction1 - prediction2) + margin; lossPrediction1.elem(arma::find(lossPrediction1 >= 0)).ones(); lossPrediction1.elem(arma::find(lossPrediction1 < 0)).zeros(); - MatType lossPrediction2 = lossPrediction1; + LossType lossPrediction2 = lossPrediction1; lossPrediction1 = -target % lossPrediction1; lossPrediction2 = target % lossPrediction2; loss = arma::join_cols(lossPrediction1, lossPrediction2); @@ -70,9 +76,9 @@ void MarginRankingLossType::Backward( loss = loss / target.n_elem; } -template +template template -void MarginRankingLossType::serialize( +void MarginRankingLoss::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/mean_absolute_percentage_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_absolute_percentage_error.hpp index ed3abef2c9..d6eb6e5e89 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_absolute_percentage_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_absolute_percentage_error.hpp @@ -37,17 +37,22 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class MeanAbsolutePercentageErrorType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MeanAbsolutePercentageError { public: /** - * Create the MeanAbsolutePercentageErrorType object. + * Create the MeanAbsolutePercentageError object. */ - MeanAbsolutePercentageErrorType(); + MeanAbsolutePercentageError(); /** * Computes the mean absolute percentage error function. @@ -56,8 +61,9 @@ class MeanAbsolutePercentageErrorType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -67,19 +73,26 @@ class MeanAbsolutePercentageErrorType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } /** - * Serialize the layer. - */ + * Serialize the layer. + */ template - void serialize(Archive& ar, const unsigned int /* version */) { } -}; // class MeanAbsolutePercentageErrorType + void serialize(Archive& ar, const unsigned int /* version */); -// Default typedef for typical `arma::mat` usage. -typedef MeanAbsolutePercentageErrorType MeanAbsolutePercentageError; + private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class MeanAbsolutePercentageError } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/mean_absolute_percentage_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_absolute_percentage_error_impl.hpp index 8185de852d..b573654e7f 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_absolute_percentage_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_absolute_percentage_error_impl.hpp @@ -18,32 +18,45 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -MeanAbsolutePercentageErrorType::MeanAbsolutePercentageErrorType() +template +MeanAbsolutePercentageError:: +MeanAbsolutePercentageError() { // Nothing to do here. } -template -typename MatType::elem_type MeanAbsolutePercentageErrorType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +MeanAbsolutePercentageError::Forward( + const PredictionType& prediction, + const TargetType& target) { - MatType loss = arma::abs((prediction - target) / target); + PredictionType loss = arma::abs((prediction - target) / target); return arma::accu(loss) * (100 / target.n_cols); } -template -void MeanAbsolutePercentageErrorType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void MeanAbsolutePercentageError::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss = (((arma::conv_to::from(prediction < target) * -2) + 1) / target) * (100 / target.n_cols); } +template +template +void MeanAbsolutePercentageError::serialize( + Archive& /* ar */, + const unsigned int /* version */) +{ + // Nothing to do here. +} + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp index 6c8324f8b5..fb84b2ba4c 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp @@ -18,18 +18,23 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The mean bias error performance function measures the network's performance - * according to the mean of errors. + * The mean bias error performance function measures the network's + * performance according to the mean of errors. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class MeanBiasErrorType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MeanBiasError { public: /** - * Create the MeanBiasErrorType object. + * Create the MeanBiasError object. * * @param reduction Specifies the reduction to apply to * the output. If false, 'mean' reduction @@ -39,7 +44,7 @@ class MeanBiasErrorType * is used and the output will be summed. * It is set to true by default. */ - MeanBiasErrorType(const bool reduction = true); + MeanBiasError(const bool reduction = true); /** * Computes the mean bias error function. @@ -48,8 +53,9 @@ class MeanBiasErrorType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -59,9 +65,15 @@ class MeanBiasErrorType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -76,12 +88,12 @@ class MeanBiasErrorType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class MeanBiasErrorType - -// Default typedef for typical `arma::mat` usage. -typedef MeanBiasErrorType MeanBiasError; +}; // class MeanBiasError } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp index 6473eda8e0..fa801ff275 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp @@ -19,20 +19,22 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -MeanBiasErrorType::MeanBiasErrorType(const bool reduction) : - reduction(reduction) +template +MeanBiasError:: + MeanBiasError(const bool reduction) : reduction(reduction) { // Nothing to do here } -template -typename MatType::elem_type MeanBiasErrorType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +MeanBiasError::Forward( + const PredictionType& prediction, + const TargetType& target) { - MatType loss = target - prediction; - typename MatType::elem_type lossSum = arma::accu(loss); + PredictionType loss = target - prediction; + typename PredictionType::elem_type lossSum = arma::accu(loss); if (reduction) return lossSum; @@ -40,11 +42,12 @@ typename MatType::elem_type MeanBiasErrorType::Forward( return lossSum / target.n_elem; } -template -void MeanBiasErrorType::Backward( - const MatType& prediction, - const MatType& /* target */, - MatType& loss) +template +template +void MeanBiasError::Backward( + const PredictionType& prediction, + const TargetType& /* target */, + LossType& loss) { loss.set_size(arma::size(prediction)); loss.fill(-1.0); @@ -53,9 +56,9 @@ void MeanBiasErrorType::Backward( loss = loss / loss.n_elem; } -template +template template -void MeanBiasErrorType::serialize( +void MeanBiasError::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index 6065951f38..6006d76d12 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -21,15 +21,21 @@ namespace ann /** Artificial Neural Network. */ { * The mean squared error performance function measures the network's * performance according to the mean of squared errors. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam ActivationFunction Activation function used for the embedding layer. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class MeanSquaredErrorType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MeanSquaredError { public: /** - * Create the MeanSquaredErrorType object. + * Create the MeanSquaredError object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -37,7 +43,7 @@ class MeanSquaredErrorType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - MeanSquaredErrorType(const bool reduction = true); + MeanSquaredError(const bool reduction = true); /** * Computes the mean squared error function. @@ -46,8 +52,9 @@ class MeanSquaredErrorType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -57,10 +64,16 @@ class MeanSquaredErrorType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } @@ -74,12 +87,12 @@ class MeanSquaredErrorType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class MeanSquaredErrorType - -// Default typedef for typical `arma::mat` usage. -typedef MeanSquaredErrorType MeanSquaredError; +}; // class MeanSquaredError } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp index c2938a6e3a..3bd64bc638 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp @@ -18,19 +18,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -MeanSquaredErrorType::MeanSquaredErrorType(const bool reduction) : - reduction(reduction) +template +MeanSquaredError + ::MeanSquaredError(const bool reduction) : reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type MeanSquaredErrorType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +MeanSquaredError::Forward( + const PredictionType& prediction, + const TargetType& target) { - typename MatType::elem_type lossSum = + typename PredictionType::elem_type lossSum = arma::accu(arma::square(prediction - target)); if (reduction) @@ -39,11 +41,12 @@ typename MatType::elem_type MeanSquaredErrorType::Forward( return lossSum / target.n_elem; } -template -void MeanSquaredErrorType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void MeanSquaredError::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss = 2 * (prediction - target); @@ -51,9 +54,9 @@ void MeanSquaredErrorType::Backward( loss = loss / target.n_elem; } -template +template template -void MeanSquaredErrorType::serialize( +void MeanSquaredError::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp index 833066f876..bdd702ef54 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp @@ -18,18 +18,23 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The mean squared logarithmic error performance function measures the - * network's performance according to the mean of squared logarithmic errors. + * The mean squared logarithmic error performance function measures the network's + * performance according to the mean of squared logarithmic errors. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class MeanSquaredLogarithmicErrorType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MeanSquaredLogarithmicError { public: /** - * Create the MeanSquaredLogarithmicErrorType object. + * Create the MeanSquaredLogarithmicError object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -37,7 +42,7 @@ class MeanSquaredLogarithmicErrorType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - MeanSquaredLogarithmicErrorType(const bool reduction = true); + MeanSquaredLogarithmicError(const bool reduction = true); /** * Computes the mean squared logarithmic error function. @@ -46,8 +51,9 @@ class MeanSquaredLogarithmicErrorType * function. * @param target The target vector. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -57,9 +63,15 @@ class MeanSquaredLogarithmicErrorType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -74,12 +86,12 @@ class MeanSquaredLogarithmicErrorType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class MeanSquaredLogarithmicErrorType - -// Default typedef for typical `arma::mat` usage. -typedef MeanSquaredLogarithmicErrorType MeanSquaredLogarithmicError; +}; // class MeanSquaredLogarithmicError } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp index a59d71a7bc..4b1d3740c7 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp @@ -18,20 +18,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -MeanSquaredLogarithmicErrorType::MeanSquaredLogarithmicErrorType( - const bool reduction) : - reduction(reduction) +template +MeanSquaredLogarithmicError +::MeanSquaredLogarithmicError(const bool reduction) : reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type MeanSquaredLogarithmicErrorType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +MeanSquaredLogarithmicError::Forward( + const PredictionType& prediction, + const TargetType& target) { - typename MatType::elem_type lossSum = + typename PredictionType::elem_type lossSum = arma::accu(arma::square(arma::log(1.0 + target) - arma::log(1.0 + prediction))); @@ -41,11 +42,12 @@ typename MatType::elem_type MeanSquaredLogarithmicErrorType::Forward( return lossSum / target.n_elem; } -template -void MeanSquaredLogarithmicErrorType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void MeanSquaredLogarithmicError::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss = 2 * (arma::log(1. + prediction) - arma::log(1. + target)) / (1. + prediction); @@ -54,9 +56,9 @@ void MeanSquaredLogarithmicErrorType::Backward( loss = loss / target.n_elem; } -template +template template -void MeanSquaredLogarithmicErrorType::serialize( +void MeanSquaredLogarithmicError::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp index 008cf09a9c..e707410155 100644 --- a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp @@ -22,21 +22,20 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The Multi-label Soft Margin Loss function. - * - * It is a criterion that optimizes a multi-label one-versus-all loss based on - * max-entropy, between input x and target y of size (N, C) where N is the - * batch size and C is the number of classes. - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class MultiLabelSoftMarginLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MultiLabelSoftMarginLoss { public: /** - * Create the MultiLabelSoftMarginLossType object. + * Create the MultiLabelSoftMarginLoss object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -46,10 +45,8 @@ class MultiLabelSoftMarginLossType * @param weights A manual rescaling weight given to each class. It is a * (1, numClasses) row vector. */ - MultiLabelSoftMarginLossType( - const bool reduction = true, - const arma::Row& weights = - arma::Row()); + MultiLabelSoftMarginLoss(const bool reduction = true, + const arma::rowvec& weights = arma::rowvec()); /** * Computes the Multi Label Soft Margin Loss function. @@ -57,8 +54,9 @@ class MultiLabelSoftMarginLossType * @param input Input data used for evaluating the specified function. * @param target The target vector with same shape as input. */ - typename MatType::elem_type Forward(const MatType& input, - const MatType& target); + template + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -67,20 +65,20 @@ class MultiLabelSoftMarginLossType * @param target The target vector. * @param output The calculated error. */ - void Backward(const MatType& input, - const MatType& target, - MatType& output); + template + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the weights assigned to each class. - const arma::Row& ClassWeights() const - { - return classWeights; - } + const arma::rowvec& ClassWeights() const { return classWeights; } //! Modify the weights assigned to each class. - arma::Row& ClassWeights() - { - return classWeights; - } + arma::rowvec& ClassWeights() { return classWeights; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -95,18 +93,18 @@ class MultiLabelSoftMarginLossType void serialize(Archive& ar, const unsigned int /* version */); private: - //! The boolean value that tells if reduction is sum or mean. + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; //! A (1, numClasses) shaped vector with weights for each class. - arma::Row classWeights; + arma::rowvec classWeights; // An internal parameter used during initialisation of class weights. bool weighted; -}; // class MultiLabelSoftMarginLossType - -// Default typedef for typical `arma::mat` usage. -typedef MultiLabelSoftMarginLossType MultiLabelSoftMarginLoss; +}; // class MultiLabelSoftMarginLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss_impl.hpp index 453254f100..a497a591d3 100644 --- a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss_impl.hpp @@ -18,10 +18,11 @@ namespace mlpack { namespace ann /** Artifical Neural Network. */ { -template -MultiLabelSoftMarginLossType::MultiLabelSoftMarginLossType( +template +MultiLabelSoftMarginLoss:: +MultiLabelSoftMarginLoss( const bool reduction, - const arma::Row& weights) : + const arma::rowvec& weights) : reduction(reduction), weighted(false) { @@ -32,9 +33,11 @@ MultiLabelSoftMarginLossType::MultiLabelSoftMarginLossType( } } -template -typename MatType::elem_type MultiLabelSoftMarginLossType::Forward( - const MatType& input, const MatType& target) +template +template +typename InputType::elem_type +MultiLabelSoftMarginLoss::Forward( + const InputType& input, const TargetType& target) { if (!weighted) { @@ -42,9 +45,9 @@ typename MatType::elem_type MultiLabelSoftMarginLossType::Forward( weighted = true; } - MatType logSigmoid = arma::log((1 / (1 + arma::exp(-input)))); - MatType logSigmoidNeg = arma::log(1 / (1 + arma::exp(input))); - MatType loss = arma::mean(arma::sum(-(target % logSigmoid + + InputType logSigmoid = arma::log((1 / (1 + arma::exp(-input)))); + InputType logSigmoidNeg = arma::log(1 / (1 + arma::exp(input))); + InputType loss = arma::mean(arma::sum(-(target % logSigmoid + (1 - target) % logSigmoidNeg)) % classWeights, 1); if (reduction) @@ -53,14 +56,15 @@ typename MatType::elem_type MultiLabelSoftMarginLossType::Forward( return arma::as_scalar(loss / input.n_rows); } -template -void MultiLabelSoftMarginLossType::Backward( - const MatType& input, - const MatType& target, - MatType& output) +template +template +void MultiLabelSoftMarginLoss::Backward( + const InputType& input, + const TargetType& target, + OutputType& output) { output.set_size(size(input)); - MatType sigmoid = (1 / (1 + arma::exp(-input))); + InputType sigmoid = (1 / (1 + arma::exp(-input))); output = -(target % (1 - sigmoid) - (1 - target) % sigmoid) % arma::repmat(classWeights, target.n_rows, 1) / output.n_elem; @@ -68,9 +72,9 @@ void MultiLabelSoftMarginLossType::Backward( output = output * input.n_rows; } -template +template template -void MultiLabelSoftMarginLossType::serialize( +void MultiLabelSoftMarginLoss::serialize( Archive& ar, const unsigned int /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp index f78a875268..49e8d06957 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -2,7 +2,7 @@ * @file methods/ann/loss_functions/negative_log_likelihood.hpp * @author Marcus Edel * - * Definition of the NegativeLogLikelihoodType class. + * Definition of the NegativeLogLikelihood class. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -20,18 +20,23 @@ namespace ann /** Artificial Neural Network. */ { /** * Implementation of the negative log likelihood layer. The negative log * likelihood layer expects that the input contains log-probabilities for each - * class. The layer also expects a class index in the range [0, numClasses - 1] - * number of classes, as target when calling the Forward function. + * class. The layer also expects a class index, in the range between 0 and + * number of classes -1, as target when calling the Forward function. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class NegativeLogLikelihoodType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class NegativeLogLikelihood { public: /** - * Create the NegativeLogLikelihoodTypeLayer object. + * Create the NegativeLogLikelihoodLayer object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -39,7 +44,7 @@ class NegativeLogLikelihoodType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - NegativeLogLikelihoodType(const bool reduction = true); + NegativeLogLikelihood(const bool reduction = true); /** * Computes the Negative log likelihood. @@ -49,8 +54,9 @@ class NegativeLogLikelihoodType * @param target The target vector, that contains the class index in the range * between 1 and the number of classes. */ - double Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. The negative log @@ -64,9 +70,25 @@ class NegativeLogLikelihoodType * between 1 and the number of classes. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the input parameter. + InputDataType& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -81,12 +103,18 @@ class NegativeLogLikelihoodType void serialize(Archive& /* ar */, const uint32_t /* version */); private: + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class NegativeLogLikelihoodType - -// Default typedef for typical `arma::mat` usage. -typedef NegativeLogLikelihoodType NegativeLogLikelihood; +}; // class NegativeLogLikelihood } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp index 82c258f838..b33979d130 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp @@ -2,7 +2,7 @@ * @file methods/ann/loss_functions/negative_log_likelihood_impl.hpp * @author Marcus Edel * - * Implementation of the NegativeLogLikelihoodType class. + * Implementation of the NegativeLogLikelihood class. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -18,19 +18,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -NegativeLogLikelihoodType::NegativeLogLikelihoodType( - const bool reduction) : reduction(reduction) +template +NegativeLogLikelihood + ::NegativeLogLikelihood(const bool reduction) : reduction(reduction) { // Nothing to do here. } -template -double NegativeLogLikelihoodType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +NegativeLogLikelihood::Forward( + const PredictionType& prediction, + const TargetType& target) { - typedef typename MatType::elem_type ElemType; + typedef typename PredictionType::elem_type ElemType; ElemType lossSum = 0; for (size_t i = 0; i < prediction.n_cols; ++i) { @@ -46,13 +48,14 @@ double NegativeLogLikelihoodType::Forward( return lossSum / target.n_elem; } -template -void NegativeLogLikelihoodType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void NegativeLogLikelihood::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { - loss = arma::zeros(prediction.n_rows, prediction.n_cols); + loss = arma::zeros(prediction.n_rows, prediction.n_cols); for (size_t i = 0; i < prediction.n_cols; ++i) { Log::Assert(target(i) >= 0 && target(i) < prediction.n_rows, @@ -65,9 +68,9 @@ void NegativeLogLikelihoodType::Backward( loss = loss / target.n_elem; } -template +template template -void NegativeLogLikelihoodType::serialize( +void NegativeLogLikelihood::serialize( Archive& ar, const uint32_t /* version */) { ar(CEREAL_NVP(reduction)); diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp index b06f366958..e3ba107423 100644 --- a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp @@ -2,7 +2,7 @@ * @file methods/ann/loss_functions/poisson_nll_loss.hpp * @author Mrityunjay Tripathi * - * Definition of the PoissonNLLLossType class. It is the negative log likelihood of + * Definition of the PoissonNLLLoss class. It is the negative log likelihood of * the Poisson distribution. * * mlpack is free software; you may redistribute it and/or modify it under the @@ -20,18 +20,24 @@ namespace ann /** Artificial Neural Network. */ { /** * Implementation of the Poisson negative log likelihood loss. This loss - * function expects input for each class. It also expects a class index, in the - * range [0, numClasses - 1], as target when calling the Forward function. + * function expects input for each class. It also expects a class index, + * in the range between 1 and the number of classes, as target when calling + * the Forward function. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class PoissonNLLLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class PoissonNLLLoss { public: /** - * Create the PoissonNLLLossType object. + * Create the PoissonNLLLoss object. * * @param logInput If true the loss is computed as * \f$ \exp(input) - target \cdot input \f$, if false then the loss is @@ -45,9 +51,9 @@ class PoissonNLLLossType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - PoissonNLLLossType(const bool logInput = true, + PoissonNLLLoss(const bool logInput = true, const bool full = false, - const typename MatType::elem_type eps = 1e-08, + const typename InputDataType::elem_type eps = 1e-08, const bool reduction = true); /** @@ -58,8 +64,9 @@ class PoissonNLLLossType * @param target The target vector, that contains the class index in the range * between 1 and the number of classes. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename InputDataType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. The Poisson Negative Log @@ -73,9 +80,20 @@ class PoissonNLLLossType * between 1 and the number of classes. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the input parameter. + InputDataType& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the value of logInput. logInput is a boolean value that tells if //! logits are given as input. @@ -93,17 +111,16 @@ class PoissonNLLLossType //! Get the value of eps. eps is a small value required to prevent 0 in //! logarithms and denominators. - typename MatType::elem_type Eps() const { return eps; } + typename InputDataType::elem_type Eps() const { return eps; } //! Modify the value of eps. eps is a small value required to prevent 0 in //! logarithms and denominators. - typename MatType::elem_type& Eps() { return eps; } + typename InputDataType::elem_type& Eps() { return eps; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } - /** * Serialize the layer. */ @@ -123,6 +140,12 @@ class PoissonNLLLossType } } + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if logits are given as input. bool logInput; @@ -131,14 +154,12 @@ class PoissonNLLLossType bool full; //! eps is a small value required to prevent 0 in logarithms and denominators. - typename MatType::elem_type eps; + typename InputDataType::elem_type eps; //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class PoissonNLLLossType -// Default typedef for typical `arma::mat` usage. -typedef PoissonNLLLossType PoissonNLLLoss; +}; // class PoissonNLLLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp index c23d0d0ad8..3152d73f56 100644 --- a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp @@ -2,7 +2,7 @@ * @file methods/ann/loss_functions/poisson_nll_loss_impl.hpp * @author Mrityunjay Tripathi * - * Implementation of the PoissonNLLLossType class. + * Implementation of the PoissonNLLLoss class. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -19,12 +19,12 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -PoissonNLLLossType::PoissonNLLLossType( +template +PoissonNLLLoss::PoissonNLLLoss( const bool logInput, const bool full, - const typename MatType::elem_type eps, - const bool reduction) : + const typename InputDataType::elem_type eps, + const bool reduction): logInput(logInput), full(full), eps(eps), @@ -33,12 +33,14 @@ PoissonNLLLossType::PoissonNLLLossType( Log::Assert(eps >= 0, "Epsilon (eps) must be greater than or equal to zero."); } -template -typename MatType::elem_type PoissonNLLLossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename InputDataType::elem_type +PoissonNLLLoss::Forward( + const PredictionType& prediction, + const TargetType& target) { - MatType loss(arma::size(prediction)); + PredictionType loss(arma::size(prediction)); if (logInput) loss = arma::exp(prediction) - target % prediction; @@ -51,11 +53,11 @@ typename MatType::elem_type PoissonNLLLossType::Forward( if (full) { const auto mask = target > 1.0; - const MatType approx = target % arma::log(target) - target + const PredictionType approx = target % arma::log(target) - target + 0.5 * arma::log(2 * M_PI * target); loss.elem(arma::find(mask)) += approx.elem(arma::find(mask)); } - typename MatType::elem_type lossSum = arma::accu(loss); + typename PredictionType::elem_type lossSum = arma::accu(loss); if (reduction) return lossSum; @@ -63,11 +65,12 @@ typename MatType::elem_type PoissonNLLLossType::Forward( return lossSum / loss.n_elem; } -template -void PoissonNLLLossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void PoissonNLLLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss.set_size(size(prediction)); @@ -80,9 +83,9 @@ void PoissonNLLLossType::Backward( loss = loss / loss.n_elem; } -template +template template -void PoissonNLLLossType::serialize( +void PoissonNLLLoss::serialize( Archive& ar, const uint32_t /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp index 15a290fb8c..454c22cb31 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -23,19 +23,22 @@ namespace ann /** Artificial Neural Network. */ { * performance equal to the negative log probability of the target with * the input distribution. * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). * @tparam DistType The type of distribution parametrized by the input. */ -template< - typename MatType = arma::mat, - typename DistType = BernoulliDistribution +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename DistType = BernoulliDistribution > -class ReconstructionLossType +class ReconstructionLoss { public: /** - * Create the ReconstructionLossType object. + * Create the ReconstructionLoss object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -43,7 +46,7 @@ class ReconstructionLossType * 'sum' reduction is used and the output will be summed. It * is set to true by default. */ - ReconstructionLossType(const bool reduction = true); + ReconstructionLoss(const bool reduction = true); /** * Computes the reconstruction loss. @@ -52,8 +55,9 @@ class ReconstructionLossType * function. * @param target The target matrix. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -63,9 +67,15 @@ class ReconstructionLossType * @param target The target matrix. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -83,12 +93,12 @@ class ReconstructionLossType //! Locally-stored distribution object. DistType dist; + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class ReconstructionLossType - -// Default typedef for typical `arma::mat` usage. -typedef ReconstructionLossType ReconstructionLoss; +}; // class ReconstructionLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp index f1927c1d0c..ccb70b90f0 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp @@ -18,20 +18,24 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -ReconstructionLossType::ReconstructionLossType( - const bool reduction) : - reduction(reduction) +template +ReconstructionLoss< + InputDataType, + OutputDataType, + DistType +>::ReconstructionLoss(const bool reduction) : reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type ReconstructionLossType::Forward( - const MatType& prediction, const MatType& target) +template +template +typename PredictionType::elem_type +ReconstructionLoss::Forward( + const PredictionType& prediction, const TargetType& target) { dist = DistType(prediction); - typename MatType::elem_type lossSum = -dist.LogProbability(target); + typename PredictionType::elem_type lossSum = -dist.LogProbability(target); if (reduction) return lossSum; @@ -39,11 +43,12 @@ typename MatType::elem_type ReconstructionLossType::Forward( return lossSum / target.n_elem; } -template -void ReconstructionLossType::Backward( - const MatType& /* prediction */, - const MatType& target, - MatType& loss) +template +template +void ReconstructionLoss::Backward( + const PredictionType& /* prediction */, + const TargetType& target, + LossType& loss) { dist.LogProbBackward(target, loss); loss *= -1; @@ -52,13 +57,12 @@ void ReconstructionLossType::Backward( loss = loss / target.n_elem; } -template +template template -void ReconstructionLossType::serialize( - Archive& ar, +void ReconstructionLoss::serialize( + Archive& ar, const uint32_t /* version */) { - ar(CEREAL_NVP(dist)); ar(CEREAL_NVP(reduction)); } diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index 6c66efc947..04f9419521 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -19,7 +19,7 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The SigmoidCrossEntropyErrorType performance function measures the network's + * The SigmoidCrossEntropyError performance function measures the network's * performance according to the cross-entropy function between the input and * target distributions. This function calculates the cross entropy * given the real values instead of providing the sigmoid activations. @@ -40,23 +40,28 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class SigmoidCrossEntropyErrorType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class SigmoidCrossEntropyError { public: /** - * Create the SigmoidCrossEntropyErrorType object. + * Create the SigmoidCrossEntropyError object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be * divided by the number of elements in the output. If true, * 'sum' reduction is used and the output will be summed. It * is set to true by default. - */ - SigmoidCrossEntropyErrorType(const bool reduction = true); + */ + SigmoidCrossEntropyError(const bool reduction = true); /** * Computes the Sigmoid CrossEntropy Error functions. @@ -65,9 +70,10 @@ class SigmoidCrossEntropyErrorType * function. * @param target The target vector. */ - inline typename MatType::elem_type Forward( - const MatType& prediction, - const MatType& target); + template + inline typename PredictionType::elem_type Forward( + const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -77,9 +83,15 @@ class SigmoidCrossEntropyErrorType * @param target The target vector. * @param loss The calculated error. */ - inline void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + inline void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -94,12 +106,12 @@ class SigmoidCrossEntropyErrorType void serialize(Archive& ar, const uint32_t /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class SigmoidCrossEntropyErrorType - -// Default typedef for typical `arma::mat` usage. -typedef SigmoidCrossEntropyErrorType SigmoidCrossEntropyError; +}; // class SigmoidCrossEntropy } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp index 7526495579..e29e4a7478 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp @@ -20,21 +20,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template -SigmoidCrossEntropyErrorType::SigmoidCrossEntropyErrorType( - const bool reduction) : - reduction(reduction) +template +SigmoidCrossEntropyError +::SigmoidCrossEntropyError(const bool reduction): reduction(reduction) { // Nothing to do here. } -template -inline typename MatType::elem_type -SigmoidCrossEntropyErrorType::Forward( - const MatType& prediction, - const MatType& target) +template +template +inline typename PredictionType::elem_type +SigmoidCrossEntropyError::Forward( + const PredictionType& prediction, + const TargetType& target) { - typedef typename MatType::elem_type ElemType; + typedef typename PredictionType::elem_type ElemType; ElemType maximum = 0; for (size_t i = 0; i < prediction.n_elem; ++i) { @@ -50,11 +50,12 @@ SigmoidCrossEntropyErrorType::Forward( return lossSum / target.n_elem; } -template -inline void SigmoidCrossEntropyErrorType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +inline void SigmoidCrossEntropyError::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss = 1.0 / (1.0 + arma::exp(-prediction)) - target; @@ -62,10 +63,10 @@ inline void SigmoidCrossEntropyErrorType::Backward( loss = loss / target.n_elem; } -template +template template -void SigmoidCrossEntropyErrorType::serialize( - Archive& ar, +void SigmoidCrossEntropyError::serialize( + Archive& ar , const uint32_t /* version */) { ar(CEREAL_NVP(reduction)); diff --git a/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp index 41d6c4d62f..a44e321fcd 100644 --- a/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp @@ -22,21 +22,20 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The Soft Margin Loss function. - * - * It is a criterion that optimizes a two-class classification logistic loss, - * between input x and target y, both having the same shape, with the target - * containing only the values 1 or -1. - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class SoftMarginLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class SoftMarginLoss { public: /** - * Create the SoftMarginLossType object. + * Create the SoftMarginLoss object. * * @param reduction Specifies the reduction to apply to the output. If false, * 'mean' reduction is used, where sum of the output will be @@ -44,7 +43,7 @@ class SoftMarginLossType * true, 'sum' reduction is used and the output will be * summed. It is set to true by default. */ - SoftMarginLossType(const bool reduction = true); + SoftMarginLoss(const bool reduction = true); /** * Computes the Soft Margin Loss function. @@ -53,8 +52,9 @@ class SoftMarginLossType * function. * @param target The target vector with same shape as input. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -64,9 +64,15 @@ class SoftMarginLossType * @param target The target vector. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the reduction type, represented as boolean //! (false 'mean' reduction, true 'sum' reduction). @@ -81,12 +87,12 @@ class SoftMarginLossType void serialize(Archive& ar, const uint32_t version); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; -}; // class SoftMarginLossType - -// Default typedef for typical `arma::mat` usage. -typedef SoftMarginLossType SoftMarginLoss; +}; // class SoftMarginLoss } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/soft_margin_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/soft_margin_loss_impl.hpp index 36908ac99e..40453564a3 100644 --- a/src/mlpack/methods/ann/loss_functions/soft_margin_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/soft_margin_loss_impl.hpp @@ -18,19 +18,21 @@ namespace mlpack { namespace ann /** Artifical Neural Network. */ { -template -SoftMarginLossType:: -SoftMarginLossType(const bool reduction) : reduction(reduction) +template +SoftMarginLoss:: +SoftMarginLoss(const bool reduction) : reduction(reduction) { // Nothing to do here. } -template -typename MatType::elem_type SoftMarginLossType::Forward( - const MatType& prediction, const MatType& target) +template +template +typename PredictionType::elem_type +SoftMarginLoss::Forward( + const PredictionType& prediction, const TargetType& target) { - MatType loss = arma::log(1 + arma::exp(-target % prediction)); - typename MatType::elem_type lossSum = arma::accu(loss); + PredictionType loss = arma::log(1 + arma::exp(-target % prediction)); + typename PredictionType::elem_type lossSum = arma::accu(loss); if (reduction) return lossSum; @@ -38,25 +40,26 @@ typename MatType::elem_type SoftMarginLossType::Forward( return lossSum / prediction.n_elem; } -template -void SoftMarginLossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template +void SoftMarginLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { loss.set_size(size(prediction)); - MatType temp = arma::exp(-target % prediction); - MatType numerator = -target % temp; - MatType denominator = 1 + temp; + PredictionType temp = arma::exp(-target % prediction); + PredictionType numerator = -target % temp; + PredictionType denominator = 1 + temp; loss = numerator / denominator; if (!reduction) loss = loss / prediction.n_elem; } -template +template template -void SoftMarginLossType::serialize( +void SoftMarginLoss::serialize( Archive& ar, const uint32_t /* version */) { ar(CEREAL_NVP(reduction)); diff --git a/src/mlpack/methods/ann/loss_functions/triplet_margin_loss.hpp b/src/mlpack/methods/ann/loss_functions/triplet_margin_loss.hpp index 3f3a6dffb9..980d863d17 100644 --- a/src/mlpack/methods/ann/loss_functions/triplet_margin_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/triplet_margin_loss.hpp @@ -24,34 +24,38 @@ namespace ann /** Artificial Neural Network. */ { * of the positive (truthy) and negative (falsy) inputs. * The distance between two samples A and B is defined as square of L2 norm * of A-B. - * + * * For more information, refer the following paper. * * @code * @article{Schroff2015, * author = {Florian Schroff, Dmitry Kalenichenko, James Philbin}, - * title = {FaceNet: A Unified Embedding for Face Recognition and - * Clustering}, + * title = {FaceNet: A Unified Embedding for Face Recognition and Clustering}, * year = {2015}, * url = {https://arxiv.org/abs/1503.03832}, * } * @endcode * - * @tparam MatType Matrix representation to accept as input and use for - * computation. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). */ -template -class TripletMarginLossType +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class TripletMarginLoss { public: /** - * Create the TripletMarginLossType object. + * Create the TripletMarginLoss object. * * @param margin The minimum value by which the distance between * Anchor and Negative sample exceeds the distance * between Anchor and Positive sample. */ - TripletMarginLossType(const double margin = 1.0); + TripletMarginLoss(const double margin = 1.0); /** * Computes the Triplet Margin Loss function. @@ -59,9 +63,9 @@ class TripletMarginLossType * @param prediction Concatenated anchor and positive sample. * @param target The negative sample. */ - typename MatType::elem_type Forward(const MatType& prediction, - const MatType& target); - + template + typename PredictionType::elem_type Forward(const PredictionType& prediction, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. * @@ -69,9 +73,15 @@ class TripletMarginLossType * @param target The negative sample. * @param loss The calculated error. */ - void Backward(const MatType& prediction, - const MatType& target, - MatType& loss); + template + void Backward(const PredictionType& prediction, + const TargetType& target, + LossType& loss); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } //! Get the value of margin. double Margin() const { return margin; } @@ -85,12 +95,12 @@ class TripletMarginLossType void serialize(Archive& ar, const unsigned int /* version */); private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + //! The margin value used in calculating Triplet Margin Loss. double margin; -}; // class TripletMarginLoss - -// Default typedef for typical `arma::mat` usage. -typedef TripletMarginLossType TripletMarginLoss; +}; // class TripletLossMargin } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/loss_functions/triplet_margin_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/triplet_margin_loss_impl.hpp index 3ad7083726..2a43bc4ac4 100644 --- a/src/mlpack/methods/ann/loss_functions/triplet_margin_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/triplet_margin_loss_impl.hpp @@ -19,42 +19,49 @@ namespace mlpack { namespace ann /** Artifical Neural Network. */ { -template -TripletMarginLossType::TripletMarginLossType(const double margin) : - margin(margin) +template +TripletMarginLoss::TripletMarginLoss( + const double margin) : margin(margin) { // Nothing to do here. } -template -typename MatType::elem_type TripletMarginLossType::Forward( - const MatType& prediction, - const MatType& target) +template +template +typename PredictionType::elem_type +TripletMarginLoss::Forward( + const PredictionType& prediction, + const TargetType& target) { - MatType anchor = + PredictionType anchor = prediction.submat(0, 0, prediction.n_rows / 2 - 1, prediction.n_cols - 1); - MatType positive = + PredictionType positive = prediction.submat(prediction.n_rows / 2, 0, prediction.n_rows - 1, prediction.n_cols - 1); return std::max(0.0, arma::accu(arma::pow(anchor - positive, 2)) - arma::accu(arma::pow(anchor - target, 2)) + margin) / anchor.n_cols; } -template -void TripletMarginLossType::Backward( - const MatType& prediction, - const MatType& target, - MatType& loss) +template +template < + typename PredictionType, + typename TargetType, + typename LossType +> +void TripletMarginLoss::Backward( + const PredictionType& prediction, + const TargetType& target, + LossType& loss) { - MatType positive = + PredictionType positive = prediction.submat(prediction.n_rows / 2, 0, prediction.n_rows - 1, prediction.n_cols - 1); loss = 2 * (target - positive) / target.n_cols; } -template +template template -void TripletMarginLossType::serialize( +void TripletMarginLoss::serialize( Archive& ar, const unsigned int /* version */) { diff --git a/src/mlpack/methods/ann/loss_functions/vr_class_reward_impl.hpp b/src/mlpack/methods/ann/loss_functions/vr_class_reward_impl.hpp deleted file mode 100644 index 761f8529a3..0000000000 --- a/src/mlpack/methods/ann/loss_functions/vr_class_reward_impl.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/** - * @file methods/ann/loss_functions/vr_class_reward_impl.hpp - * @author Marcus Edel - * - * Implementation of the VRClassRewardType class, which implements the variance - * reduced classification reinforcement layer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTIONS_VR_CLASS_REWARD_IMPL_HPP -#define MLPACK_METHODS_ANN_LOSS_FUNCTIONS_VR_CLASS_REWARD_IMPL_HPP - -// In case it hasn't yet been included. -#include "vr_class_reward.hpp" - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -template -VRClassRewardType::VRClassRewardType( - const double scale, - const bool sizeAverage) : - scale(scale), - sizeAverage(sizeAverage), - reward(0) -{ - // Nothing to do here. -} - -template -typename MatType::elem_type VRClassRewardType::Forward( - const MatType& input, const MatType& target) -{ - double output = 0; - for (size_t i = 0; i < input.n_cols - 1; ++i) - { - const size_t currentTarget = target(i); - Log::Assert(currentTarget < input.n_rows, "Target class out of range."); - - output -= input(currentTarget, i); - } - - reward = 0; - arma::uword index = 0; - - for (size_t i = 0; i < input.n_cols - 1; ++i) - { - input.unsafe_col(i).max(index); - reward = (index == target(i)) * scale; - } - - if (sizeAverage) - { - return output - reward / (input.n_cols - 1); - } - - return output - reward; -} - -template -void VRClassRewardType::Backward( - const MatType& input, - const MatType& target, - MatType& output) -{ - output = arma::zeros(input.n_rows, input.n_cols); - for (size_t i = 0; i < (input.n_cols - 1); ++i) - { - const size_t currentTarget = target(i); - Log::Assert(currentTarget < input.n_rows, "Target class out of range."); - - output(currentTarget, i) = -1; - } - - double vrReward = reward - input(0, 1); - if (sizeAverage) - { - vrReward /= input.n_cols - 1; - } - - const double norm = sizeAverage ? 2.0 / (input.n_cols - 1) : 2.0; - - output(0, 1) = norm * (input(0, 1) - reward); - network.back()->Reward() = vrReward; -} - -template -template -void VRClassRewardType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(CEREAL_NVP(scale)); - ar(CEREAL_NVP(sizeAverage)); - ar(CEREAL_NVP(reward)); - ar(CEREAL_NVP(network)); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/make_alias.hpp b/src/mlpack/methods/ann/make_alias.hpp deleted file mode 100644 index 65b68565d3..0000000000 --- a/src/mlpack/methods/ann/make_alias.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/** - * @file make_alias.hpp - * @author Ryan Curtin - * - * Implementation of `MakeAlias()`, a utility function. This is meant to be - * used in `SetWeights()` calls in various layers, to wrap internal weight - * objects as aliases around the given memory pointers. - */ -#ifndef MLPACK_METHODS_ANN_MAKE_ALIAS_HPP -#define MLPACK_METHODS_ANN_MAKE_ALIAS_HPP - -#include - -namespace mlpack { -namespace ann { - -/** - * Reconstruct `m` as an alias around the memory `newMem`, with size `numRows` x - * `numCols`. - */ -template -void MakeAlias(MatType& m, - typename MatType::elem_type* newMem, - const size_t numRows, - const size_t numCols) -{ - // We use placement new to reinitialize the object, since the copy and move - // assignment operators in Armadillo will end up copying memory instead of - // making an alias. - m.~MatType(); - new (&m) MatType(newMem, numRows, numCols, false, true); -} - -/** - * Reconstruct `c` as an alias around the memory` newMem`, with size `numRows` x - * `numCols` x `numSlices`. - */ -template -void MakeAlias(CubeType& c, - typename CubeType::elem_type* newMem, - const size_t numRows, - const size_t numCols, - const size_t numSlices) -{ - // We use placement new to reinitialize the object, since the copy and move - // assignment operators in Armadillo will end up copying memory instead of - // making an alias. - c.~CubeType(); - new (&c) CubeType(newMem, numRows, numCols, numSlices, false, true); -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/not_adapted/rbm/CMakeLists.txt b/src/mlpack/methods/ann/rbm/CMakeLists.txt similarity index 100% rename from src/mlpack/methods/ann/not_adapted/rbm/CMakeLists.txt rename to src/mlpack/methods/ann/rbm/CMakeLists.txt diff --git a/src/mlpack/methods/ann/not_adapted/rbm/rbm.hpp b/src/mlpack/methods/ann/rbm/rbm.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/rbm/rbm.hpp rename to src/mlpack/methods/ann/rbm/rbm.hpp diff --git a/src/mlpack/methods/ann/not_adapted/rbm/rbm_impl.hpp b/src/mlpack/methods/ann/rbm/rbm_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/rbm/rbm_impl.hpp rename to src/mlpack/methods/ann/rbm/rbm_impl.hpp diff --git a/src/mlpack/methods/ann/not_adapted/rbm/rbm_policies.hpp b/src/mlpack/methods/ann/rbm/rbm_policies.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/rbm/rbm_policies.hpp rename to src/mlpack/methods/ann/rbm/rbm_policies.hpp diff --git a/src/mlpack/methods/ann/not_adapted/rbm/spike_slab_rbm_impl.hpp b/src/mlpack/methods/ann/rbm/spike_slab_rbm_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/not_adapted/rbm/spike_slab_rbm_impl.hpp rename to src/mlpack/methods/ann/rbm/spike_slab_rbm_impl.hpp diff --git a/src/mlpack/methods/ann/regularizer/lregularizer.hpp b/src/mlpack/methods/ann/regularizer/lregularizer.hpp index 41538ae752..96a88e2959 100644 --- a/src/mlpack/methods/ann/regularizer/lregularizer.hpp +++ b/src/mlpack/methods/ann/regularizer/lregularizer.hpp @@ -44,7 +44,7 @@ class LRegularizer template void Evaluate(const MatType& weight, MatType& gradient); - //! Serialize the regularizer. + //! Serialize the regularizer (nothing to do). template void serialize(Archive& ar, const uint32_t /* version */); diff --git a/src/mlpack/methods/ann/regularizer/no_regularizer.hpp b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp index 3a4f321c1f..920c8d3057 100644 --- a/src/mlpack/methods/ann/regularizer/no_regularizer.hpp +++ b/src/mlpack/methods/ann/regularizer/no_regularizer.hpp @@ -44,12 +44,6 @@ class NoRegularizer { // Nothing to do here. } - - template - void serialize(Archive& /* ar */, const uint32_t /* version */) - { - // Nothing to do. - } }; } // namespace ann diff --git a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp index eacc07e20a..3e56521165 100644 --- a/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp +++ b/src/mlpack/methods/ann/regularizer/orthogonal_regularizer.hpp @@ -55,7 +55,7 @@ class OrthogonalRegularizer template void Evaluate(const MatType& weight, MatType& gradient); - //! Serialize the regularizer. + //! Serialize the regularizer (nothing to do). template void serialize(Archive& ar, const uint32_t /* version */); diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp index 9fb3f14203..949aecee9b 100644 --- a/src/mlpack/methods/ann/rnn.hpp +++ b/src/mlpack/methods/ann/rnn.hpp @@ -13,32 +13,43 @@ #define MLPACK_METHODS_ANN_RNN_HPP #include -#include -#include "ffn.hpp" +#include "visitor/delete_visitor.hpp" +#include "visitor/delta_visitor.hpp" +#include "visitor/output_parameter_visitor.hpp" +#include "visitor/reset_visitor.hpp" + +#include "init_rules/network_init.hpp" + +#include +#include +#include +#include + +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Definition of a standard recurrent neural network container. A recurrent - * neural network can handle recurrent layers (i.e. `RecurrentLayer`s), which - * hold internal state and are passed sequences of data as inputs. - * - * As opposed to the standard `FFN`, which takes data in a matrix format where - * each column is a data point, the `RNN` takes a cube format where each column - * is a data point and each slice is a time step. + * Implementation of a standard recurrent neural network container. * * @tparam OutputLayerType The output layer type used to evaluate the network. * @tparam InitializationRuleType Rule used to initialize the weight matrix. */ template< - typename OutputLayerType = NegativeLogLikelihood, - typename InitializationRuleType = RandomInitialization, - typename MatType = arma::mat> + typename OutputLayerType = NegativeLogLikelihood<>, + typename InitializationRuleType = RandomInitialization, + typename... CustomLayers +> class RNN { public: + //! Convenience typedef for the internal model construction. + using NetworkType = RNN; + /** * Create the RNN object. * @@ -48,67 +59,63 @@ class RNN * If you want to pass in a parameter and discard the original parameter * object, be sure to use std::move to avoid unnecessary copy. * - * @param bpttSteps Number of time steps to use for BPTT (backpropagation - * through time) when training. - * @param single If true, then the network will expect only a single timestep - * for responses. (That is, every input sequence only has one single - * output; so, `responses.n_slices` should be 1 when calling `Train()`.) + * @param rho Maximum number of steps to backpropagate through time (BPTT). + * @param single Predict only the last element of the input sequence. * @param outputLayer Output layer used to evaluate the network. * @param initializeRule Optional instantiated InitializationRule object * for initializing the network parameter. */ - RNN(const size_t bpttSteps = 0, + RNN(const size_t rho, const bool single = false, OutputLayerType outputLayer = OutputLayerType(), InitializationRuleType initializeRule = InitializationRuleType()); //! Copy constructor. RNN(const RNN&); + //! Move constructor. RNN(RNN&&); - //! Copy operator. + + //! Copy assignment operator. RNN& operator=(const RNN&); - //! Move assignment operator. + + //! Move assignment operator RNN& operator=(RNN&&); - //! Destroy the RNN and release any memory it is holding. + //! Destructor to release allocated memory. ~RNN(); /** - * Add a new module to the model. + * Check if the optimizer has MaxIterations() parameter, if it does + * then check if it's value is less than the number of datapoints + * in the dataset. * - * @param args The layer parameter. + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. */ - template - void Add(Args... args) { network.template Add(args...); } + template + typename std::enable_if< + HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; /** - * Add a new module to the model. + * Check if the optimizer has MaxIterations() parameter, if it + * doesn't then simply return from the function. * - * @param layer The Layer to be added to the model. + * @tparam OptimizerType Type of optimizer to use to train the model. + * @param optimizer optimizer used in the training process. + * @param samples Number of datapoints in the dataset. */ - void Add(Layer* layer) { network.Add(layer); } - - //! Get the network model. - const std::vector*>& Network() const - { - return network.Network().Network(); - } + template + typename std::enable_if< + !HasMaxIterations + ::value, void>::type + WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const; /** - * Modify the network model. Be careful! If you change the structure of the - * network or parameters for layers, its state may become invalid, and the - * next time it is used for any operation the parameters will be reset. - * - * Don't add any layers like this; use `Add()` instead. - */ - std::vector*>& Network() - { - return network.Network().Network(); - } - - /** - * Train the recurrent network on the given input data using the given + * Train the recurrent neural network on the given input data using the given * optimizer. * * This will use the existing model parameters as a starting point for the @@ -118,6 +125,13 @@ class RNN * If you want to pass in a parameter and discard the original parameter * object, be sure to use std::move to avoid unnecessary copy. * + * The format of the data should be as follows: + * - each slice should correspond to a time step + * - each column should correspond to a data point + * - each row should correspond to a dimension + * So, e.g., predictors(i, j, k) is the i'th dimension of the j'th data point + * at time slice k. + * * @tparam OptimizerType Type of optimizer to use to train the model. * @tparam CallbackTypes Types of Callback Functions. * @param predictors Input training variables. @@ -128,16 +142,15 @@ class RNN * @return The final objective of the trained model (NaN or Inf on error). */ template - typename MatType::elem_type Train( - arma::Cube predictors, - arma::Cube responses, - OptimizerType& optimizer, - CallbackTypes&&... callbacks); + double Train(arma::cube predictors, + arma::cube responses, + OptimizerType& optimizer, + CallbackTypes&&... callbacks); /** - * Train the recurrent network on the given input data. By default, the - * RMSProp optimization algorithm is used, but others can be specified - * (such as ens::SGD). + * Train the recurrent neural network on the given input data. By default, the + * SGD optimization algorithm is used, but others can be specified + * (such as ens::RMSprop). * * This will use the existing model parameters as a starting point for the * optimization. If this is not what you want, then you should access the @@ -146,19 +159,25 @@ class RNN * If you want to pass in a parameter and discard the original parameter * object, be sure to use std::move to avoid unnecessary copy. * + * The format of the data should be as follows: + * - each slice should correspond to a time step + * - each column should correspond to a data point + * - each row should correspond to a dimension + * So, e.g., predictors(i, j, k) is the i'th dimension of the j'th data point + * at time slice k. + * * @tparam OptimizerType Type of optimizer to use to train the model. - * @param predictors Input training variables. * @tparam CallbackTypes Types of Callback Functions. + * @param predictors Input training variables. * @param responses Outputs results from input training variables. * @param callbacks Callback function for ensmallen optimizer `OptimizerType`. * See https://www.ensmallen.org/docs.html#callback-documentation. * @return The final objective of the trained model (NaN or Inf on error). */ - template - typename MatType::elem_type Train( - arma::Cube predictors, - arma::Cube responses, - CallbackTypes&&... callbacks); + template + double Train(arma::cube predictors, + arma::cube responses, + CallbackTypes&&... callbacks); /** * Predict the responses to a given set of predictors. The responses will @@ -168,101 +187,42 @@ class RNN * If you want to pass in a parameter and discard the original parameter * object, be sure to use std::move to avoid unnecessary copy. * + * The format of the data should be as follows: + * - each slice should correspond to a time step + * - each column should correspond to a data point + * - each row should correspond to a dimension + * So, e.g., predictors(i, j, k) is the i'th dimension of the j'th data point + * at time slice k. The responses will be in the same format. + * * @param predictors Input predictors. * @param results Matrix to put output predictions of responses into. - * @param batchSize Batch size to use for prediction. + * @param batchSize Number of points to predict at once. */ - void Predict(arma::Cube predictors, - arma::Cube& results, - const size_t batchSize = 128); - - // Return the nujmber of weights in the model. - size_t WeightSize() { return network.WeightSize(); } + void Predict(arma::cube predictors, + arma::cube& results, + const size_t batchSize = 256); /** - * Set the logical dimensions of the input. `Train()` and `Predict()` expect - * data to be passed such that one point corresponds to one column, but this - * data is allowed to be an arbitrary higher-order tensor. - * - * So, if the input is meant to be 28x28x3 images, then the input data to - * `Train()` or `Predict()` should have 28*28*3 = 2352 rows, and - * `InputDImensions()` should be set to `{ 28, 28, 3}`. Then, the layers of - * the network will interpret each input point as a 3-dimensional image - * instead of a 1-dimensional vector. - * - * If `InputDimensions()` is left unset before training, the data will be - * assumed to be a 1-dimensional vector. - */ - std::vector& InputDimensions() { return network.InputDimensions(); } - //! Get the logical dimensions of the input. - const std::vector& InputDimensions() const - { - return network.InputDimensions(); - } - - //! Return the initial point for the optimization. - const MatType& Parameters() const { return network.Parameters(); } - //! Modify the initial point for the optimization. - MatType& Parameters() { return network.Parameters(); } - - //! Return the number of steps allowed for BPTT. - size_t BPTTSteps() const { return bpttSteps; } - //! Modify the number of steps allowed for BPTT. - size_t& BPTTSteps() { return bpttSteps; } - - /** - * Reset the stored data of the network entirely. This reset all weights of - * each layer using `InitializationRuleType`, and prepares the network to - * accept a (flat 1-d) input size of `inputDimensionality` (if passed), or - * whatever input size has been set with `InputDimensions()`. - * - * This also resets the mode of the network to prediction mode (not training - * mode). See `SetNetworkMode()` for more information. - */ - void Reset(const size_t inputDimensionality = 0); - - /** - * Set all the layers in the network to training mode, if `training` is - * `true`, or set all the layers in the network to testing mode, if `training` - * is `false`. - */ - void SetNetworkMode(const bool training) { network.SetNetworkMode(training); } - - /** - * Evaluate the recurrent network with the given predictors and responses. - * This functions is usually used to monitor progress while training. - * - * @param predictors Input variables. - * @param responses Target outputs for input variables. - */ - typename MatType::elem_type Evaluate( - const arma::Cube& predictors, - const arma::Cube& responses); - - //! Serialize the model. - template - void serialize(Archive& ar, const uint32_t /* version */); - - // - // Only ensmallen utility functions for training are found below here. - // They generally aren't useful otherwise. - // - - /** - * Evaluate the recurrent network with the given parameters. This function - * is usually called by the optimizer to train the model. + * Evaluate the recurrent neural network with the given parameters. This + * function is usually called by the optimizer to train the model. * * @param parameters Matrix model parameters. + * @param begin Index of the starting point to use for objective function + * evaluation. + * @param batchSize Number of points to be passed at a time to use for + * objective function evaluation. + * @param deterministic Whether or not to train or test the model. Note some + * layer act differently in training or testing mode. */ - typename MatType::elem_type Evaluate(const MatType& parameters); + double Evaluate(const arma::mat& parameters, + const size_t begin, + const size_t batchSize, + const bool deterministic); - /** - * Evaluate the recurrent network with the given parameters, but using only - * a number of data points. This is useful for optimizers such as SGD, which - * require a separable objective function. - * - * Note that the network may return different results depending on the mode it - * is in (see `SetNetworkMode()`). + /** + * Evaluate the recurrent neural network with the given parameters. This + * function is usually called by the optimizer to train the model. This just + * calls the other overload of Evaluate() with deterministic = true. * * @param parameters Matrix model parameters. * @param begin Index of the starting point to use for objective function @@ -270,26 +230,13 @@ class RNN * @param batchSize Number of points to be passed at a time to use for * objective function evaluation. */ - typename MatType::elem_type Evaluate(const MatType& parameters, - const size_t begin, - const size_t batchSize); + double Evaluate(const arma::mat& parameters, + const size_t begin, + const size_t batchSize); /** - * Evaluate the recurrent network with the given parameters. - * This function is usually called by the optimizer to train the model. - * This just calls the overload of EvaluateWithGradient() with batchSize = 1. - * - * @param parameters Matrix model parameters. - * @param gradient Matrix to output gradient into. - */ - template - typename MatType::elem_type EvaluateWithGradient(const MatType& parameters, - GradType& gradient); - - /** - * Evaluate the recurrent network with the given parameters, but using only - * a number of data points. This is useful for optimizers such as SGD, which - * require a separable objective function. + * Evaluate the recurrent neural network with the given parameters. This + * function is usually called by the optimizer to train the model. * * @param parameters Matrix model parameters. * @param begin Index of the starting point to use for objective function @@ -299,15 +246,16 @@ class RNN * objective function evaluation. */ template - typename MatType::elem_type EvaluateWithGradient(const MatType& parameters, - const size_t begin, - GradType& gradient, - const size_t batchSize); + double EvaluateWithGradient(const arma::mat& parameters, + const size_t begin, + GradType& gradient, + const size_t batchSize); /** - * Evaluate the gradient of the recurrent network with the given parameters, - * and with respect to only a number of points in the dataset. This is useful - * for optimizers such as SGD, which require a separable objective function. + * Evaluate the gradient of the recurrent neural network with the given + * parameters, and with respect to only one point in the dataset. This is + * useful for optimizers such as SGD, which require a separable objective + * function. * * @param parameters Matrix of the model parameters to be optimized. * @param begin Index of the starting point to use for objective function @@ -316,69 +264,191 @@ class RNN * @param batchSize Number of points to be processed as a batch for objective * function gradient evaluation. */ - template - void Gradient(const MatType& parameters, + void Gradient(const arma::mat& parameters, const size_t begin, - GradType& gradient, + arma::mat& gradient, const size_t batchSize); - //! Return the number of separable functions (the number of predictor points). - size_t NumFunctions() const { return predictors.n_cols; } - /** - * Note: this function is implement so that it can be used by ensmallen's - * optimizers. It's not generally meant to be used otherwise. - * - * Shuffle the order of function visitation. (This is equivalent to shuffling - * the dataset during training.) + * Shuffle the order of function visitation. This may be called by the + * optimizer. */ void Shuffle(); - /** - * Prepare the network for the given data. - * This function won't actually trigger training process. + /* + * Add a new module to the model. * - * @param predictors Input data variables. - * @param responses Outputs results from input data variables. + * @param args The layer parameter. */ - void ResetData(arma::Cube predictors, - arma::Cube responses); + template + void Add(Args... args) { network.push_back(new LayerType(args...)); } + + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + + //! Return the number of separable functions (the number of predictor points). + size_t NumFunctions() const { return numFunctions; } + + //! Return the initial point for the optimization. + const arma::mat& Parameters() const { return parameter; } + //! Modify the initial point for the optimization. + arma::mat& Parameters() { return parameter; } + + //! Return the maximum length of backpropagation through time. + const size_t& Rho() const { return rho; } + //! Modify the maximum length of backpropagation through time. + size_t& Rho() { return rho; } + + //! Get the matrix of responses to the input data points. + const arma::cube& Responses() const { return responses; } + //! Modify the matrix of responses to the input data points. + arma::cube& Responses() { return responses; } + + //! Get the matrix of data points (predictors). + const arma::cube& Predictors() const { return predictors; } + //! Modify the matrix of data points (predictors). + arma::cube& Predictors() { return predictors; } + + /** + * Reset the state of the network. This ensures that all internally-held + * gradients are set to 0, all memory cells are reset, and the parameters + * matrix is the right size. + */ + void Reset(); + + /** + * Reset the module information (weights/parameters). + */ + void ResetParameters(); + + //! Serialize the model. + template + void serialize(Archive& ar, const uint32_t /* version */); private: // Helper functions. + /** + * The Forward algorithm (part of the Forward-Backward algorithm). Computes + * forward probabilities for each module. + * + * @param input Data sequence to compute probabilities for. + */ + template + void Forward(const InputType& input); /** - * Iterate over all layers and reset the recurrent layers' states. Prepare - * each recurrent layer to store up to `memorySize` previous states, operating - * with a batch size of `batchSize`. + * Reset the state of RNN cells in the network for new input sequence. */ - void ResetMemoryState(const size_t memorySize, const size_t batchSize); + void ResetCells(); - //! Set the previous step index of all recurrent layers to `step`. - void SetPreviousStep(const size_t step); - //! Set the current step index of all recurrent layers to `step`. - void SetCurrentStep(const size_t step); + /** + * The Backward algorithm (part of the Forward-Backward algorithm). Computes + * backward pass for module. + */ + void Backward(); - //! Number of timesteps to consider for backpropagation through time (BPTT). - size_t bpttSteps; - //! Whether the network expects only one single response per sequence, or one - //! response per time step. + /** + * Iterate through all layer modules and update the the gradient using the + * layer defined optimizer. + */ + template + void Gradient(const InputType& input); + + /** + * Reset the module status by setting the current deterministic parameter + * for all modules that implement the Deterministic function. + */ + void ResetDeterministic(); + + /** + * Reset the gradient for all modules that implement the Gradient function. + */ + void ResetGradients(arma::mat& gradient); + + //! Number of steps to backpropagate through time (BPTT). + size_t rho; + + //! Instantiated outputlayer used to evaluate the network. + OutputLayerType outputLayer; + + //! Instantiated InitializationRule object for initializing the network + //! parameter. + InitializationRuleType initializeRule; + + //! The input size. + size_t inputSize; + + //! The output size. + size_t outputSize; + + //! The target size. + size_t targetSize; + + //! Indicator if we already trained the model. + bool reset; + + //! Only predict the last element of the input sequence. bool single; - //! The network itself is stored in this FFN object. Note that this network - //! may contain recursive layers, and thus we will be responsible for - //! occasionally resetting any memory cells. - FFN network; + //! Locally-stored model modules. + std::vector > network; - //! The matrix of data points (predictors). This member is empty, except - //! during training---we must store a local copy of the training data since - //! the ensmallen optimizer will not provide training data. - arma::Cube predictors; + //! The matrix of data points (predictors). + arma::cube predictors; - //! The matrix of responses to the input data points. This member is empty, - //! except during training. - arma::Cube responses; -}; // class RNNType + //! The matrix of responses to the input data points. + arma::cube responses; + + //! Matrix of (trained) parameters. + arma::mat parameter; + + //! The number of separable functions (the number of predictor points). + size_t numFunctions; + + //! The current error for the backward pass. + arma::mat error; + + //! Locally-stored delta visitor. + DeltaVisitor deltaVisitor; + + //! Locally-stored output parameter visitor. + OutputParameterVisitor outputParameterVisitor; + + //! List of all module parameters for the backward pass (BBTT). + std::vector moduleOutputParameter; + + //! Locally-stored weight size visitor. + WeightSizeVisitor weightSizeVisitor; + + //! Locally-stored copy visitor + CopyVisitor copyVisitor; + + //! Locally-stored reset visitor. + ResetVisitor resetVisitor; + + //! Locally-stored delete visitor. + DeleteVisitor deleteVisitor; + + //! The current evaluation mode (training or testing). + bool deterministic; + + //! The current gradient for the gradient pass. + arma::mat currentGradient; + + // The BRN class should have access to internal members. + template< + typename OutputLayerType1, + typename MergeLayerType1, + typename MergeOutputType1, + typename InitializationRuleType1, + typename... CustomLayers1 + > + friend class BRNN; +}; // class RNN } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp index a276af3e91..a7e7c65f3e 100644 --- a/src/mlpack/methods/ann/rnn_impl.hpp +++ b/src/mlpack/methods/ann/rnn_impl.hpp @@ -14,603 +14,621 @@ // In case it hasn't been included yet. #include "rnn.hpp" -#include "layer/recurrent_layer.hpp" + +#include "visitor/load_output_parameter_visitor.hpp" +#include "visitor/save_output_parameter_visitor.hpp" +#include "visitor/forward_visitor.hpp" +#include "visitor/backward_visitor.hpp" +#include "visitor/reset_cell_visitor.hpp" +#include "visitor/deterministic_set_visitor.hpp" +#include "visitor/gradient_set_visitor.hpp" +#include "visitor/gradient_visitor.hpp" +#include "visitor/weight_set_visitor.hpp" + +#include "util/check_input_shape.hpp" namespace mlpack { namespace ann /** Artificial Neural Network. */ { -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::RNN( - const size_t bpttSteps, +template +RNN::RNN( + const size_t rho, const bool single, OutputLayerType outputLayer, InitializationRuleType initializeRule) : - bpttSteps(bpttSteps), + rho(rho), + outputLayer(std::move(outputLayer)), + initializeRule(std::move(initializeRule)), + inputSize(0), + outputSize(0), + targetSize(0), + reset(false), single(single), - network(std::move(outputLayer), std::move(initializeRule)) + numFunctions(0), + deterministic(true) { /* Nothing to do here */ } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::RNN( +template +RNN::RNN( const RNN& network) : - bpttSteps(network.bpttSteps), + rho(network.rho), + outputLayer(network.outputLayer), + initializeRule(network.initializeRule), + inputSize(network.inputSize), + outputSize(network.outputSize), + targetSize(network.targetSize), + reset(network.reset), single(network.single), - network(network.network) + parameter(network.parameter), + numFunctions(network.numFunctions), + deterministic(network.deterministic) { - // Nothing else to do. + for (size_t i = 0; i < network.network.size(); ++i) + { + this->network.push_back(boost::apply_visitor(copyVisitor, + network.network[i])); + boost::apply_visitor(resetVisitor, this->network.back()); + } } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::RNN( +template +RNN::RNN( RNN&& network) : - bpttSteps(std::move(network.bpttSteps)), + rho(std::move(network.rho)), + outputLayer(std::move(network.outputLayer)), + initializeRule(std::move(network.initializeRule)), + inputSize(std::move(network.inputSize)), + outputSize(std::move(network.outputSize)), + targetSize(std::move(network.targetSize)), + reset(std::move(network.reset)), single(std::move(network.single)), - network(std::move(network.network)) + network(std::move(network.network)), + parameter(std::move(network.parameter)), + numFunctions(std::move(network.numFunctions)), + deterministic(std::move(network.deterministic)) { // Nothing to do here. } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -RNN< - OutputLayerType, - InitializationRuleType, - MatType ->& -RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::operator=(const RNN& other) +template +RNN::~RNN() { - if (this != &other) + for (LayerTypes& layer : network) { - bpttSteps = other.bpttSteps; - single = other.single; - network = other.network; - predictors.clear(); - responses.clear(); + boost::apply_visitor(deleteVisitor, layer); } - - return *this; } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -RNN< - OutputLayerType, - InitializationRuleType, - MatType ->& -RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::operator=(RNN&& other) +template +template +typename std::enable_if< + HasMaxIterations + ::value, void>::type +RNN:: +WarnMessageMaxIterations(OptimizerType& optimizer, size_t samples) const { - if (this != &other) + if (optimizer.MaxIterations() < samples && + optimizer.MaxIterations() != 0) { - bpttSteps = std::move(other.bpttSteps); - single = std::move(other.single); - network = std::move(other.network); - predictors.clear(); - responses.clear(); + Log::Warn << "The optimizer's maximum number of iterations " + << "is less than the size of the dataset; the " + << "optimizer will not pass over the entire " + << "dataset. To fix this, modify the maximum " + << "number of iterations to be at least equal " + << "to the number of points of your dataset " + << "(" << samples << ")." << std::endl; } - - return *this; } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::~RNN() +template +template +typename std::enable_if< + !HasMaxIterations + ::value, void>::type +RNN:: +WarnMessageMaxIterations(OptimizerType& /* optimizer */, + size_t /* samples */) const { - // Nothing special to do. + return; } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> +template template -typename MatType::elem_type RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::Train( - arma::Cube predictors, - arma::Cube responses, +double RNN::Train( + arma::cube predictors, + arma::cube responses, OptimizerType& optimizer, CallbackTypes&&... callbacks) { - ResetData(std::move(predictors), std::move(responses)); + CheckInputShape > >( + network, predictors.n_rows, "RNN<>::Train()"); - network.WarnMessageMaxIterations(optimizer, this->predictors.n_cols); + numFunctions = responses.n_cols; - // Ensure that the network can be used. - network.CheckNetwork("RNN::Train()", this->predictors.n_rows, true, true); + this->predictors = std::move(predictors); + this->responses = std::move(responses); + + this->deterministic = true; + ResetDeterministic(); + + if (!reset) + { + ResetParameters(); + } + + WarnMessageMaxIterations(optimizer, this->predictors.n_cols); // Train the model. - Timer::Start("rnn_optimization"); - const typename MatType::elem_type out = - optimizer.Optimize(*this, network.Parameters(), callbacks...); - Timer::Stop("rnn_optimization"); + const double out = optimizer.Optimize(*this, parameter, callbacks...); - Log::Info << "RNN::Train(): final objective of trained model is " << out + Log::Info << "RNN::RNN(): final objective of trained model is " << out << "." << std::endl; return out; } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -template -typename MatType::elem_type RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::Train( - arma::Cube predictors, - arma::Cube responses, - CallbackTypes&&... callbacks) +template +void RNN::ResetCells() { - OptimizerType optimizer; - return Train(std::move(predictors), std::move(responses), optimizer, - callbacks...); + for (size_t i = 1; i < network.size(); ++i) + { + boost::apply_visitor(ResetCellVisitor(rho), network[i]); + } } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -void RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::Predict( - arma::Cube predictors, - arma::Cube& results, - const size_t batchSize) +template +template +double RNN::Train( + arma::cube predictors, + arma::cube responses, + CallbackTypes&&... callbacks) { - // Ensure that the network is configured correctly. - network.CheckNetwork("RNN::Predict()", predictors.n_rows, true, false); + CheckInputShape > >( + network, predictors.n_rows, "RNN<>::Train()"); - results.set_size(network.network.OutputSize(), predictors.n_cols, - predictors.n_slices); + numFunctions = responses.n_cols; - MatType inputAlias, outputAlias; - for (size_t i = 0; i < predictors.n_cols; i += batchSize) + this->predictors = std::move(predictors); + this->responses = std::move(responses); + + this->deterministic = true; + ResetDeterministic(); + + if (!reset) + { + ResetParameters(); + } + + OptimizerType optimizer; + + WarnMessageMaxIterations(optimizer, this->predictors.n_cols); + + // Train the model. + const double out = optimizer.Optimize(*this, parameter, callbacks...); + + Log::Info << "RNN::RNN(): final objective of trained model is " << out + << "." << std::endl; + return out; +} + +template +void RNN::Predict( + arma::cube predictors, arma::cube& results, const size_t batchSize) +{ + CheckInputShape > >( + network, predictors.n_rows, "RNN<>::Predict()"); + + ResetCells(); + + if (parameter.is_empty()) + { + ResetParameters(); + } + + if (!deterministic) + { + deterministic = true; + ResetDeterministic(); + } + + const size_t effectiveBatchSize = std::min(batchSize, + size_t(predictors.n_cols)); + + Forward(arma::mat(predictors.slice(0).colptr(0), predictors.n_rows, + effectiveBatchSize, false, true)); + arma::mat resultsTemp = boost::apply_visitor(outputParameterVisitor, + network.back()); + + outputSize = resultsTemp.n_rows; + results = arma::zeros(outputSize, predictors.n_cols, rho); + results.slice(0).submat(0, 0, results.n_rows - 1, + effectiveBatchSize - 1) = resultsTemp; + + // Process in accordance with the given batch size. + for (size_t begin = 0; begin < predictors.n_cols; begin += batchSize) { const size_t effectiveBatchSize = std::min(batchSize, - size_t(predictors.n_cols) - i); - - // Since we aren't doing a backward pass, we don't actually need to store - // the state for each time step---we can fit it all in one buffer. - ResetMemoryState(1, effectiveBatchSize); - SetPreviousStep(size_t(-1)); - SetCurrentStep(size_t(0)); - - // Iterate over all time steps. - for (size_t t = 0; t < predictors.n_slices; ++t) + size_t(predictors.n_cols - begin)); + for (size_t seqNum = !begin; seqNum < rho; ++seqNum) { - // If it is after the first step, we have a previous state. - if (t == 1) - SetPreviousStep(size_t(0)); + Forward(arma::mat(predictors.slice(seqNum).colptr(begin), + predictors.n_rows, effectiveBatchSize, false, true)); - // Create aliases for the input and output. - MakeAlias(inputAlias, - (typename MatType::elem_type*) predictors.slice(t).colptr(i), - predictors.n_rows, effectiveBatchSize); - MakeAlias(outputAlias, results.slice(t).colptr(i), results.n_rows, - effectiveBatchSize); - - network.Forward(inputAlias, outputAlias); + results.slice(seqNum).submat(0, begin, results.n_rows - 1, begin + + effectiveBatchSize - 1) = boost::apply_visitor(outputParameterVisitor, + network.back()); } } } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -void RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::Reset(const size_t inputDimensionality) +template +double RNN::Evaluate( + const arma::mat& /* parameters */, + const size_t begin, + const size_t batchSize, + const bool deterministic) { - // This is a reimplementation of FFN::Reset() that correctly prints - // "RNN::Reset()". - network.Parameters().clear(); - - if (inputDimensionality != 0) + if (parameter.is_empty()) { - network.CheckNetwork("RNN::Reset()", inputDimensionality, true, false); + ResetParameters(); + } + + if (deterministic != this->deterministic) + { + this->deterministic = deterministic; + ResetDeterministic(); + } + + if (!inputSize) + { + inputSize = predictors.n_rows; + targetSize = responses.n_rows; + } + else if (targetSize == 0) + { + targetSize = responses.n_rows; + } + + ResetCells(); + + double performance = 0; + size_t responseSeq = 0; + + for (size_t seqNum = 0; seqNum < rho; ++seqNum) + { + // Wrap a matrix around our data to avoid a copy. + arma::mat stepData(predictors.slice(seqNum).colptr(begin), + predictors.n_rows, batchSize, false, true); + Forward(stepData); + if (!single) + { + responseSeq = seqNum; + } + + performance += outputLayer.Forward(boost::apply_visitor( + outputParameterVisitor, network.back()), + arma::mat(responses.slice(responseSeq).colptr(begin), + responses.n_rows, batchSize, false, true)); + } + + if (outputSize == 0) + { + outputSize = boost::apply_visitor(outputParameterVisitor, + network.back()).n_elem / batchSize; + } + + return performance; +} + +template +double RNN::Evaluate( + const arma::mat& parameters, + const size_t begin, + const size_t batchSize) +{ + return Evaluate(parameters, begin, batchSize, true); +} + +template +template +double RNN:: +EvaluateWithGradient(const arma::mat& /* parameters */, + const size_t begin, + GradType& gradient, + const size_t batchSize) +{ + // Initialize passed gradient. + if (gradient.is_empty()) + { + if (parameter.is_empty()) + { + ResetParameters(); + } + + gradient = arma::zeros(parameter.n_rows, parameter.n_cols); } else { - const size_t inputDims = std::accumulate(network.InputDimensions().begin(), - network.InputDimensions().end(), 0); - network.CheckNetwork("RNN::Reset()", inputDims, true, false); - } -} - -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -template -void RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(CEREAL_NVP(bpttSteps)); - ar(CEREAL_NVP(single)); - ar(CEREAL_NVP(network)); - - if (Archive::is_loading::value) - { - // We can clear these members, since it's not possible to serialize in the - // middle of training and resume. - predictors.clear(); - responses.clear(); - } -} - -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -typename MatType::elem_type RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::Evaluate( - const MatType& /* parameters */, - const size_t begin, - const size_t batchSize) -{ - // Ensure the network is valid. - network.CheckNetwork("RNN::Evaluate()", predictors.n_rows); - - // The core of the computation here is to pass through each step. Since we - // are not computing the gradient, we can be "clever" and use only one memory - // cell---we don't need to know about the past. - ResetMemoryState(1, batchSize); - SetCurrentStep(0); - SetPreviousStep(size_t(-1)); - MatType output(network.network.OutputSize(), batchSize); - - typename MatType::elem_type loss = 0.0; - MatType stepData, responseData; - for (size_t t = 0; t < predictors.n_slices; ++t) - { - if (t == 1) - SetPreviousStep(0); - - // Manually reset the data of the network to be an alias of the current time - // step. - MakeAlias(network.predictors, predictors.slice(t).colptr(begin), - predictors.n_rows, batchSize); - const size_t responseStep = (single) ? 0 : t; - MakeAlias(network.responses, responses.slice(responseStep).colptr(begin), - responses.n_rows, batchSize); - - loss += network.Evaluate(output, begin, batchSize); + gradient.zeros(); } - return loss; -} - -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -template -typename MatType::elem_type RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::EvaluateWithGradient( - const MatType& parameters, - GradType& gradient) -{ - return EvaluateWithGradient(parameters, 0, gradient, predictors.n_cols); -} - -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -template -typename MatType::elem_type RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::EvaluateWithGradient( - const MatType& /* parameters */, - const size_t begin, - GradType& gradient, - const size_t batchSize) -{ - network.CheckNetwork("RNN::EvaluateWithGradient()", predictors.n_rows); - - typename MatType::elem_type loss = 0; - - // We must save anywhere between 1 and `bpttSteps` states, but we are limited - // by `predictors.n_slices`. - const size_t effectiveBPTTSteps = std::max(size_t(1), - std::min(bpttSteps, size_t(predictors.n_slices))); - - ResetMemoryState(effectiveBPTTSteps, batchSize); - SetPreviousStep(size_t(-1)); - arma::Cube outputs( - network.network.OutputSize(), batchSize, effectiveBPTTSteps); - - // If `bpttSteps` is less than the number of time steps in the data, then for - // the first few steps, we won't actually need to hold onto any historical - // information, since BPTT will never go back that far. - const size_t extraSteps = (predictors.n_slices - effectiveBPTTSteps + 1); - MatType stepData, outputData, responseData; - for (size_t t = 0; t < std::min(size_t(predictors.n_slices), extraSteps); ++t) + if (this->deterministic) { - SetCurrentStep(0); - - // Make an alias of the step's data. - MakeAlias(stepData, predictors.slice(t).colptr(begin), predictors.n_rows, - batchSize); - MakeAlias(outputData, outputs.slice(t).memptr(), outputs.n_rows, - outputs.n_cols); - network.network.Forward(stepData, outputData); - - const size_t responseStep = (single) ? 0 : t; - MakeAlias(responseData, responses.slice(responseStep).colptr(begin), - responses.n_rows, batchSize); - - loss += network.outputLayer.Forward(outputData, responseData); - - SetPreviousStep(0); + this->deterministic = false; + ResetDeterministic(); } - // Next, we reach the time steps that will be used for BPTT, for which we must - // preserve step data. - for (size_t t = extraSteps; t < predictors.n_slices; ++t) + if (!inputSize) { - SetCurrentStep(t - extraSteps + 1); + inputSize = predictors.n_rows; + targetSize = responses.n_rows; + } + else if (targetSize == 0) + { + targetSize = responses.n_rows; + } + ResetCells(); + + double performance = 0; + size_t responseSeq = 0; + const size_t effectiveRho = std::min(rho, size_t(responses.size())); + + for (size_t seqNum = 0; seqNum < effectiveRho; ++seqNum) + { // Wrap a matrix around our data to avoid a copy. - MakeAlias(stepData, predictors.slice(t).colptr(begin), predictors.n_rows, - batchSize); - MakeAlias(outputData, outputs.slice(t).memptr(), outputs.n_rows, - outputs.n_cols); - network.network.Forward(stepData, outputData); + arma::mat stepData(predictors.slice(seqNum).colptr(begin), + predictors.n_rows, batchSize, false, true); + Forward(stepData); + if (!single) + { + responseSeq = seqNum; + } - const size_t responseStep = (single) ? 0 : t; - MakeAlias(responseData, responses.slice(responseStep).colptr(begin), - responses.n_rows, batchSize); + for (size_t l = 0; l < network.size(); ++l) + { + boost::apply_visitor(SaveOutputParameterVisitor(moduleOutputParameter), + network[l]); + } - loss += network.outputLayer.Forward(outputData, responseData); - - SetPreviousStep(t - extraSteps + 1); + performance += outputLayer.Forward(boost::apply_visitor( + outputParameterVisitor, network.back()), + arma::mat(responses.slice(responseSeq).colptr(begin), + responses.n_rows, batchSize, false, true)); } - // Add loss (this is not dependent on time steps, and should only be added - // once). - loss += network.network.Loss(); + if (outputSize == 0) + { + outputSize = boost::apply_visitor(outputParameterVisitor, + network.back()).n_elem / batchSize; + } // Initialize current/working gradient. - gradient.zeros(network.Parameters().n_rows, network.Parameters().n_cols); - GradType currentGradient; - currentGradient.zeros(network.Parameters().n_rows, - network.Parameters().n_cols); - - SetPreviousStep(size_t(-1)); - const size_t minStep = predictors.n_slices - effectiveBPTTSteps + 1; - for (size_t t = predictors.n_slices; t >= minStep; --t) + if (currentGradient.is_empty()) { - SetCurrentStep(t - 1); + currentGradient = arma::zeros(parameter.n_rows, + parameter.n_cols); + } + ResetGradients(currentGradient); + + for (size_t seqNum = 0; seqNum < effectiveRho; ++seqNum) + { currentGradient.zeros(); - MatType error(outputs.n_rows, outputs.n_cols); + for (size_t l = 0; l < network.size(); ++l) + { + boost::apply_visitor(LoadOutputParameterVisitor(moduleOutputParameter), + network[network.size() - 1 - l]); + } - // Set up the response by backpropagating through the output layer. Note - // that if we are in 'single' mode, we don't care what the network outputs - // until the input sequence is done, so there is no error for any timestep - // other than the first one. - if (single && (t - 1) < responses.n_slices - 1) + if (single && seqNum > 0) { error.zeros(); } + else if (single && seqNum == 0) + { + outputLayer.Backward(boost::apply_visitor( + outputParameterVisitor, network.back()), + arma::mat(responses.slice(0).colptr(begin), + responses.n_rows, batchSize, false, true), error); + } else { - MakeAlias(outputData, outputs.slice(t - 1).colptr(0), outputs.n_rows, - outputs.n_cols); - const size_t respStep = (single) ? 0 : t - 1; - MakeAlias(responseData, responses.slice(respStep).colptr(begin), - responses.n_rows, batchSize); - network.outputLayer.Backward(outputData, responseData, error); + outputLayer.Backward(boost::apply_visitor( + outputParameterVisitor, network.back()), + arma::mat(responses.slice(effectiveRho - seqNum - 1).colptr(begin), + responses.n_rows, batchSize, false, true), error); } - // Now pass that error backwards through the network. - MakeAlias(outputData, outputs.slice(t - 1).colptr(0), outputs.n_rows, - outputs.n_cols); - MatType networkDelta; - network.network.Backward(outputData, error, networkDelta); - - MakeAlias(stepData, predictors.slice(t - 1).colptr(begin), - predictors.n_rows, batchSize); - network.network.Gradient(stepData, error, currentGradient); + Backward(); + Gradient( + arma::mat(predictors.slice(effectiveRho - seqNum - 1).colptr(begin), + predictors.n_rows, batchSize, false, true)); gradient += currentGradient; - - SetPreviousStep(t - 1); } - return loss; + return performance; } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -template -void RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::Gradient( - const MatType& parameters, +template +void RNN::Gradient( + const arma::mat& parameters, const size_t begin, - GradType& gradient, + arma::mat& gradient, const size_t batchSize) { this->EvaluateWithGradient(parameters, begin, gradient, batchSize); } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -void RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::Shuffle() +template +void RNN::Shuffle() { - math::ShuffleData(predictors, responses, predictors, responses); + arma::cube newPredictors, newResponses; + math::ShuffleData(predictors, responses, newPredictors, newResponses); + + predictors = std::move(newPredictors); + responses = std::move(newResponses); } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -void RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::ResetData( - arma::Cube predictors, - arma::Cube responses) +template +void RNN::ResetParameters() { - this->predictors = std::move(predictors); - this->responses = std::move(responses); + ResetDeterministic(); + + // Reset the network parameter with the given initialization rule. + NetworkInitialization networkInit(initializeRule); + networkInit.Initialize(network, parameter); + + reset = true; } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -void RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::ResetMemoryState(const size_t memorySize, const size_t batchSize) +template +void RNN::Reset() { - // Iterate over all layers and set the memory size. - for (Layer* l : network.Network()) + ResetParameters(); + ResetCells(); + currentGradient.zeros(); + ResetGradients(currentGradient); +} + +template +void RNN::ResetDeterministic() +{ + DeterministicSetVisitor deterministicSetVisitor(deterministic); + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deterministicSetVisitor)); +} + +template +void RNN::ResetGradients( + arma::mat& gradient) +{ + size_t offset = 0; + for (LayerTypes& layer : network) { - // We can only call ClearRecurrentState() on RecurrentLayers. - RecurrentLayer* r = - dynamic_cast*>(l); - if (r != nullptr) - r->ClearRecurrentState(memorySize, batchSize); + offset += boost::apply_visitor(GradientSetVisitor(gradient, offset), layer); } } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -void RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::SetPreviousStep(const size_t step) +template +template +void RNN::Forward(const InputType& input) { - // Iterate over all layers and set the memory size. - for (Layer* l : network.Network()) + boost::apply_visitor(ForwardVisitor(input, + boost::apply_visitor(outputParameterVisitor, network.front())), + network.front()); + + for (size_t i = 1; i < network.size(); ++i) { - // We can only call SetPreviousStep() on RecurrentLayers. - RecurrentLayer* r = - dynamic_cast*>(l); - if (r != nullptr) - r->PreviousStep() = step; + boost::apply_visitor(ForwardVisitor( + boost::apply_visitor(outputParameterVisitor, network[i - 1]), + boost::apply_visitor(outputParameterVisitor, network[i])), + network[i]); } } -template< - typename OutputLayerType, - typename InitializationRuleType, - typename MatType -> -void RNN< - OutputLayerType, - InitializationRuleType, - MatType ->::SetCurrentStep(const size_t step) +template +void RNN::Backward() { - // Iterate over all layers and set the memory size. - for (Layer* l : network.Network()) + boost::apply_visitor(BackwardVisitor( + boost::apply_visitor(outputParameterVisitor, network.back()), + error, boost::apply_visitor(deltaVisitor, + network.back())), network.back()); + + for (size_t i = 2; i < network.size(); ++i) { - // We can only call SetPreviousStep() on RecurrentLayers. - RecurrentLayer* r = - dynamic_cast*>(l); - if (r != nullptr) - r->CurrentStep() = step; + boost::apply_visitor(BackwardVisitor( + boost::apply_visitor(outputParameterVisitor, + network[network.size() - i]), boost::apply_visitor( + deltaVisitor, network[network.size() - i + 1]), + boost::apply_visitor(deltaVisitor, network[network.size() - i])), + network[network.size() - i]); + } +} + +template +template +void RNN::Gradient(const InputType& input) +{ + boost::apply_visitor(GradientVisitor(input, + boost::apply_visitor(deltaVisitor, network[1])), network.front()); + + for (size_t i = 1; i < network.size() - 1; ++i) + { + boost::apply_visitor(GradientVisitor( + boost::apply_visitor(outputParameterVisitor, network[i - 1]), + boost::apply_visitor(deltaVisitor, network[i + 1])), + network[i]); + } +} + +template +template +void RNN::serialize( + Archive& ar, const uint32_t /* version */) +{ + ar(CEREAL_NVP(parameter)); + ar(CEREAL_NVP(rho)); + ar(CEREAL_NVP(single)); + ar(CEREAL_NVP(inputSize)); + ar(CEREAL_NVP(outputSize)); + ar(CEREAL_NVP(targetSize)); + ar(CEREAL_NVP(reset)); + + if (cereal::is_loading()) + { + std::for_each(network.begin(), network.end(), + boost::apply_visitor(deleteVisitor)); + network.clear(); + } + + ar(CEREAL_VECTOR_VARIANT_POINTER(network)); + + // If we are loading, we need to initialize the weights. + if (cereal::is_loading()) + { + size_t offset = 0; + for (LayerTypes& layer : network) + { + offset += boost::apply_visitor(WeightSetVisitor(parameter, offset), + layer); + + boost::apply_visitor(resetVisitor, layer); + } + + deterministic = true; + ResetDeterministic(); } } diff --git a/src/mlpack/methods/ann/util/CMakeLists.txt b/src/mlpack/methods/ann/util/CMakeLists.txt new file mode 100644 index 0000000000..dffec0c265 --- /dev/null +++ b/src/mlpack/methods/ann/util/CMakeLists.txt @@ -0,0 +1,14 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + check_input_shape.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mlpack/methods/ann/util/check_input_shape.hpp b/src/mlpack/methods/ann/util/check_input_shape.hpp new file mode 100644 index 0000000000..59f2c72da2 --- /dev/null +++ b/src/mlpack/methods/ann/util/check_input_shape.hpp @@ -0,0 +1,53 @@ +/** + * @file methods/ann/util/check_input_shape.hpp + * @author Khizir Siddiqui + * @author Nippun Sharma + * + * Definition of the CheckInputShape() function that checks + * whether the shape of input is consistent with the first layer + * of the neural network. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_UTIL_CHECK_INPUT_SHAPE_HPP +#define MLPACK_METHODS_ANN_UTIL_CHECK_INPUT_SHAPE_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */{ + +template +void CheckInputShape(const T& network, + const size_t inputShape, + const std::string& functionName) +{ + for (size_t l = 0; l < network.size(); ++l) + { + size_t layerInShape = boost::apply_visitor(InShapeVisitor(), network[l]); + if (layerInShape == 0) + { + continue; + } + else if (layerInShape == inputShape) + { + break; + } + else + { + std::string estr = functionName + ": the first layer of the network " + + "expects " + std::to_string(layerInShape) + " elements, but the " + + "input has " + std::to_string(inputShape) + " dimensions!"; + throw std::logic_error(estr); + } + } +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/CMakeLists.txt b/src/mlpack/methods/ann/visitor/CMakeLists.txt new file mode 100644 index 0000000000..fa207d6092 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/CMakeLists.txt @@ -0,0 +1,71 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + add_visitor.hpp + add_visitor_impl.hpp + backward_visitor.hpp + backward_visitor_impl.hpp + bias_set_visitor.hpp + bias_set_visitor_impl.hpp + copy_visitor.hpp + copy_visitor_impl.hpp + delete_visitor.hpp + delete_visitor_impl.hpp + delta_visitor.hpp + delta_visitor_impl.hpp + deterministic_set_visitor.hpp + deterministic_set_visitor_impl.hpp + forward_visitor.hpp + forward_visitor_impl.hpp + gradient_set_visitor.hpp + gradient_set_visitor_impl.hpp + gradient_update_visitor.hpp + gradient_update_visitor_impl.hpp + gradient_visitor.hpp + gradient_visitor_impl.hpp + gradient_zero_visitor.hpp + gradient_zero_visitor_impl.hpp + load_output_parameter_visitor.hpp + load_output_parameter_visitor_impl.hpp + loss_visitor.hpp + loss_visitor_impl.hpp + output_height_visitor.hpp + output_height_visitor_impl.hpp + output_parameter_visitor.hpp + output_parameter_visitor_impl.hpp + output_width_visitor.hpp + output_width_visitor_impl.hpp + parameters_set_visitor.hpp + parameters_set_visitor_impl.hpp + parameters_visitor.hpp + parameters_visitor_impl.hpp + reset_cell_visitor.hpp + reset_cell_visitor_impl.hpp + reset_visitor.hpp + reset_visitor_impl.hpp + reward_set_visitor.hpp + reward_set_visitor_impl.hpp + run_set_visitor.hpp + run_set_visitor_impl.hpp + save_output_parameter_visitor.hpp + save_output_parameter_visitor_impl.hpp + set_input_height_visitor.hpp + set_input_height_visitor_impl.hpp + set_input_width_visitor.hpp + set_input_width_visitor_impl.hpp + weight_set_visitor.hpp + weight_set_visitor_impl.hpp + weight_size_visitor.hpp + weight_size_visitor_impl.hpp + input_shape_visitor.hpp + input_shape_visitor_impl.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/methods/ann/visitor/add_visitor.hpp b/src/mlpack/methods/ann/visitor/add_visitor.hpp new file mode 100644 index 0000000000..fe8096a0ba --- /dev/null +++ b/src/mlpack/methods/ann/visitor/add_visitor.hpp @@ -0,0 +1,64 @@ +/** + * @file methods/ann/visitor/add_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Add() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_ADD_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_ADD_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * AddVisitor exposes the Add() method of the given module. + */ +template +class AddVisitor : public boost::static_visitor +{ + public: + //! Exposes the Add() method of the given module. + template + AddVisitor(T newLayer); + + //! Exposes the Add() method. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The layer that should be added. + LayerTypes newLayer; + + //! Only add the layer if the module implements the Add() function. + template + typename std::enable_if< + HasAddCheck)>::value, void>::type + LayerAdd(T* layer) const; + + //! Do not add the layer if the module doesn't implement the Add() function. + template + typename std::enable_if< + !HasAddCheck)>::value, void>::type + LayerAdd(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "add_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/add_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/add_visitor_impl.hpp new file mode 100644 index 0000000000..62725e95ed --- /dev/null +++ b/src/mlpack/methods/ann/visitor/add_visitor_impl.hpp @@ -0,0 +1,64 @@ +/** + * @file methods/ann/visitor/add_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Add() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_ADD_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_ADD_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "add_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! AddVisitor visitor class. +template +template +inline AddVisitor::AddVisitor(T newLayer) : + newLayer(std::move(newLayer)) +{ + /* Nothing to do here. */ +} + +template +template +inline void AddVisitor::operator()(LayerType* layer) const +{ + LayerAdd(layer); +} + +template +inline void AddVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +template +inline typename std::enable_if< + HasAddCheck)>::value, void>::type +AddVisitor::LayerAdd(T* layer) const +{ + layer->Add(newLayer); +} + +template +template +inline typename std::enable_if< + !HasAddCheck)>::value, void>::type +AddVisitor::LayerAdd(T* /* layer */) const +{ + /* Nothing to do here. */ +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/backward_visitor.hpp b/src/mlpack/methods/ann/visitor/backward_visitor.hpp new file mode 100644 index 0000000000..e59ff53eef --- /dev/null +++ b/src/mlpack/methods/ann/visitor/backward_visitor.hpp @@ -0,0 +1,85 @@ +/** + * @file methods/ann/visitor/backward_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Backward() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_BACKWARD_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_BACKWARD_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * BackwardVisitor executes the Backward() function given the input, error and + * delta parameter. + */ +class BackwardVisitor : public boost::static_visitor +{ + public: + //! Execute the Backward() function given the input, error and delta + //! parameter. + BackwardVisitor(const arma::mat& input, + const arma::mat& error, + arma::mat& delta); + + //! Execute the Backward() function for the layer with the specified index. + BackwardVisitor(const arma::mat& input, + const arma::mat& error, + arma::mat& delta, + const size_t index); + + //! Execute the Backward() function. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The input parameter set. + const arma::mat& input; + + //! The error parameter. + const arma::mat& error; + + //! The delta parameter. + arma::mat& delta; + + //! The index of the layer to run. + size_t index; + + //! Indicates whether to use index or not + bool hasIndex; + + //! Execute the Backward() function if the module does not have Run() + //! check. + template + typename std::enable_if< + !HasRunCheck::value, void>::type + LayerBackward(T* layer, arma::mat& input) const; + + //! Execute the Backward() function if the module is has Run() function. + template + typename std::enable_if< + HasRunCheck::value, void>::type + LayerBackward(T* layer, arma::mat& input) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "backward_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/backward_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/backward_visitor_impl.hpp new file mode 100644 index 0000000000..24f6c180d9 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/backward_visitor_impl.hpp @@ -0,0 +1,84 @@ +/** + * @file methods/ann/visitor/backward_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Backward() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_BACKWARD_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_BACKWARD_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "backward_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! BackwardVisitor visitor class. +inline BackwardVisitor::BackwardVisitor(const arma::mat& input, + const arma::mat& error, + arma::mat& delta) : + input(input), + error(error), + delta(delta), + index(0), + hasIndex(false) +{ + /* Nothing to do here. */ +} + +inline BackwardVisitor::BackwardVisitor(const arma::mat& input, + const arma::mat& error, + arma::mat& delta, + const size_t index) : + input(input), + error(error), + delta(delta), + index(index), + hasIndex(true) +{ + /* Nothing to do here. */ +} + +template +inline void BackwardVisitor::operator()(LayerType* layer) const +{ + LayerBackward(layer, layer->OutputParameter()); +} + +inline void BackwardVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasRunCheck::value, void>::type +BackwardVisitor::LayerBackward(T* layer, arma::mat& /* input */) const +{ + layer->Backward(input, error, delta); +} + +template +inline typename std::enable_if< + HasRunCheck::value, void>::type +BackwardVisitor::LayerBackward(T* layer, arma::mat& /* input */) const +{ + if (!hasIndex) + { + layer->Backward(input, error, delta); + } + else + { + layer->Backward(input, error, delta, index); + } +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp b/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp new file mode 100644 index 0000000000..c081c73a2a --- /dev/null +++ b/src/mlpack/methods/ann/visitor/bias_set_visitor.hpp @@ -0,0 +1,82 @@ +/** + * @file methods/ann/visitor/bias_set_visitor.hpp + * @author Toshal Agrawal + * + * This file provides an abstraction for the Bias() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_BIAS_SET_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_BIAS_SET_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * BiasSetVisitor updates the module bias parameters given the parameters set. + */ +class BiasSetVisitor : public boost::static_visitor +{ + public: + //! Update the bias parameters given the parameters' set and offset. + BiasSetVisitor(arma::mat& weight, const size_t offset = 0); + + //! Update the parameters' set. + template + size_t operator()(LayerType* layer) const; + + size_t operator()(MoreTypes layer) const; + + private: + //! The parameters' set. + arma::mat& weight; + + //! The parameters' offset. + const size_t offset; + + //! Do not update the bias parameters if the module doesn't implement the + //! Bias() or Model() function. + template + typename std::enable_if< + !HasBiasCheck::value && + !HasModelCheck::value, size_t>::type + LayerSize(T* layer) const; + + //! Update the bias parameters if the module implements the Model() function. + template + typename std::enable_if< + !HasBiasCheck::value && + HasModelCheck::value, size_t>::type + LayerSize(T* layer) const; + + //! Update the bias parameters if the module implements the Bias() function. + template + typename std::enable_if< + HasBiasCheck::value && + !HasModelCheck::value, size_t>::type + LayerSize(T* layer) const; + + //! Update the bias parameters if the module implements the Model() and + //! Bias() function. + template + typename std::enable_if< + HasBiasCheck::value && + HasModelCheck::value, size_t>::type + LayerSize(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "bias_set_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp new file mode 100644 index 0000000000..2eacbb61d6 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/bias_set_visitor_impl.hpp @@ -0,0 +1,101 @@ +/** + * @file methods/ann/visitor/bias_set_visitor_impl.hpp + * @author Toshal Agrawal + * + * Implementation of the Bias() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_BIAS_SET_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_BIAS_SET_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "bias_set_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! BiasSetVisitor visitor class. +inline BiasSetVisitor::BiasSetVisitor(arma::mat& weight, const size_t offset) : + weight(weight), + offset(offset) +{ + /* Nothing to do here. */ +} + +template +inline size_t BiasSetVisitor::operator()(LayerType* layer) const +{ + return LayerSize(layer); +} + +inline size_t BiasSetVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasBiasCheck::value && + !HasModelCheck::value, size_t>::type +BiasSetVisitor::LayerSize(T* /* layer */) const +{ + return 0; +} + +template +inline typename std::enable_if< + !HasBiasCheck::value && + HasModelCheck::value, size_t>::type +BiasSetVisitor::LayerSize(T* layer) const +{ + size_t modelOffset = 0; + + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(BiasSetVisitor( + weight, modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +template +inline typename std::enable_if< + HasBiasCheck::value && + !HasModelCheck::value, size_t>::type +BiasSetVisitor::LayerSize(T* layer) const +{ + layer->Bias() = arma::mat(weight.memptr() + offset, + layer->Bias().n_rows, layer->Bias().n_cols, false, false); + + return layer->Bias().n_elem; +} + +template +inline typename std::enable_if< + HasBiasCheck::value && + HasModelCheck::value, size_t>::type +BiasSetVisitor::LayerSize(T* layer) const +{ + layer->Bias() = arma::mat(weight.memptr() + offset, + layer->Bias().n_rows, layer->Bias().n_cols, false, false); + + size_t modelOffset = layer->Bias().n_elem; + + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(BiasSetVisitor( + weight, modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/copy_visitor.hpp b/src/mlpack/methods/ann/visitor/copy_visitor.hpp new file mode 100644 index 0000000000..14bf5f91ad --- /dev/null +++ b/src/mlpack/methods/ann/visitor/copy_visitor.hpp @@ -0,0 +1,41 @@ +/** + * @file methods/ann/visitor/copy_visitor.hpp + * @author Shangtong Zhang + * + * This file provides an abstraction for copy between layers. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_COPY_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_COPY_VISITOR_HPP + +#include +#include + +namespace mlpack { +namespace ann { + +/** + * This visitor is to support copy constructor for neural network module. + * We want a layer-wise copy rather than simple duplicate the pointer. + */ +template +class CopyVisitor : public boost::static_visitor > +{ + public: + template + LayerTypes operator()(LayerType*) const; + + LayerTypes operator()(MoreTypes) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation +#include "copy_visitor_impl.hpp" +#endif + diff --git a/src/mlpack/methods/ann/visitor/copy_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/copy_visitor_impl.hpp new file mode 100644 index 0000000000..d143e18799 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/copy_visitor_impl.hpp @@ -0,0 +1,39 @@ +/** + * @file methods/ann/visitor/copy_visitor_impl.hpp + * @author Shangtong Zhang + * + * This file provides an implementation for copy between layers + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_COPY_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_COPY_VISITOR_IMPL_HPP + +#include +#include + +namespace mlpack { +namespace ann { + +template +template +inline LayerTypes +CopyVisitor::operator()(LayerType* layer) const +{ + return new LayerType(*layer); +} + +template +inline LayerTypes +CopyVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/delete_visitor.hpp b/src/mlpack/methods/ann/visitor/delete_visitor.hpp new file mode 100644 index 0000000000..d65f4e3e4d --- /dev/null +++ b/src/mlpack/methods/ann/visitor/delete_visitor.hpp @@ -0,0 +1,51 @@ +/** + * @file methods/ann/visitor/delete_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Delete() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_DELETE_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_DELETE_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * DeleteVisitor executes the destructor of the instantiated object. + */ +class DeleteVisitor : public boost::static_visitor +{ + public: + //! Execute the destructor if the layer does not hold layers internally. + template + typename std::enable_if< + !HasModelCheck::value, void>::type + operator()(LayerType* layer) const; + + //! Execute the destructor if the layer does hold layers internally. + template + typename std::enable_if< + HasModelCheck::value, void>::type + operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "delete_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/delete_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/delete_visitor_impl.hpp new file mode 100644 index 0000000000..f9f43936e3 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/delete_visitor_impl.hpp @@ -0,0 +1,53 @@ +/** + * @file methods/ann/visitor/delete_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Delete() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_DELETE_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_DELETE_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "delete_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! DeleteVisitor visitor class. +template +inline typename std::enable_if< + !HasModelCheck::value, void>::type +DeleteVisitor::operator()(LayerType* layer) const +{ + if (layer) + delete layer; +} + +template +inline typename std::enable_if< + HasModelCheck::value, void>::type +DeleteVisitor::operator()(LayerType* layer) const +{ + if (layer) + { + for (size_t i = 0; i < layer->Model().size(); ++i) + boost::apply_visitor(DeleteVisitor(), layer->Model()[i]); + + delete layer; + } +} + +inline void DeleteVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/delta_visitor.hpp b/src/mlpack/methods/ann/visitor/delta_visitor.hpp new file mode 100644 index 0000000000..12ecfd2017 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/delta_visitor.hpp @@ -0,0 +1,43 @@ +/** + * @file methods/ann/visitor/delta_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Delta() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_DELTA_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_DELTA_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * DeltaVisitor exposes the delta parameter of the given module. + */ +class DeltaVisitor : public boost::static_visitor +{ + public: + //! Return the delta parameter. + template + arma::mat& operator()(LayerType* layer) const; + + arma::mat& operator()(MoreTypes layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "delta_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/delta_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/delta_visitor_impl.hpp new file mode 100644 index 0000000000..3a8e2b6b92 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/delta_visitor_impl.hpp @@ -0,0 +1,36 @@ +/** + * @file methods/ann/visitor/delta_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Delta() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_DELTA_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_DELTA_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "delta_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! DeltaVisitor visitor class. +template +inline arma::mat& DeltaVisitor::operator()(LayerType *layer) const +{ + return layer->Delta(); +} + +inline arma::mat& DeltaVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/deterministic_set_visitor.hpp b/src/mlpack/methods/ann/visitor/deterministic_set_visitor.hpp new file mode 100644 index 0000000000..f70d740e80 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/deterministic_set_visitor.hpp @@ -0,0 +1,83 @@ +/** + * @file methods/ann/visitor/deterministic_set_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Deterministic() function for + * different layers and automatically directs any parameter to the right layer + * type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_DETERMINISTIC_SET_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_DETERMINISTIC_SET_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * DeterministicSetVisitor set the deterministic parameter given the + * deterministic value. + */ +class DeterministicSetVisitor : public boost::static_visitor +{ + public: + //! Set the deterministic parameter given the current deterministic value. + DeterministicSetVisitor(const bool deterministic = true); + + //! Set the deterministic parameter. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The deterministic parameter. + const bool deterministic; + + //! Set the deterministic parameter if the module implements the + //! Deterministic() and Model() function. + template + typename std::enable_if< + HasDeterministicCheck::value && + HasModelCheck::value, void>::type + LayerDeterministic(T* layer) const; + + //! Set the deterministic parameter if the module implements the + //! Model() function. + template + typename std::enable_if< + !HasDeterministicCheck::value && + HasModelCheck::value, void>::type + LayerDeterministic(T* layer) const; + + //! Set the deterministic parameter if the module implements the + //! Deterministic() function. + template + typename std::enable_if< + HasDeterministicCheck::value && + !HasModelCheck::value, void>::type + LayerDeterministic(T* layer) const; + + //! Do not set the deterministic parameter if the module doesn't implement the + //! Deterministic() or Model() function. + template + typename std::enable_if< + !HasDeterministicCheck::value && + !HasModelCheck::value, void>::type + LayerDeterministic(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "deterministic_set_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/deterministic_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/deterministic_set_visitor_impl.hpp new file mode 100644 index 0000000000..06d8bafb03 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/deterministic_set_visitor_impl.hpp @@ -0,0 +1,88 @@ +/** + * @file methods/ann/visitor/deterministic_set_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Deterministic() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_DETERMINISTIC_SET_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_DETERMINISTIC_SET_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "deterministic_set_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! DeterministicSetVisitor visitor class. +inline DeterministicSetVisitor::DeterministicSetVisitor( + const bool deterministic) : deterministic(deterministic) +{ + /* Nothing to do here. */ +} + +template +inline void DeterministicSetVisitor::operator()(LayerType* layer) const +{ + LayerDeterministic(layer); +} + +inline void DeterministicSetVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + HasDeterministicCheck::value && + HasModelCheck::value, void>::type +DeterministicSetVisitor::LayerDeterministic(T* layer) const +{ + layer->Deterministic() = deterministic; + + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(DeterministicSetVisitor(deterministic), + layer->Model()[i]); + } +} + +template +inline typename std::enable_if< + !HasDeterministicCheck::value && + HasModelCheck::value, void>::type +DeterministicSetVisitor::LayerDeterministic(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(DeterministicSetVisitor(deterministic), + layer->Model()[i]); + } +} + +template +inline typename std::enable_if< + HasDeterministicCheck::value && + !HasModelCheck::value, void>::type +DeterministicSetVisitor::LayerDeterministic(T* layer) const +{ + layer->Deterministic() = deterministic; +} + +template +inline typename std::enable_if< + !HasDeterministicCheck::value && + !HasModelCheck::value, void>::type +DeterministicSetVisitor::LayerDeterministic(T* /* input */) const +{ + /* Nothing to do here. */ +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/forward_visitor.hpp b/src/mlpack/methods/ann/visitor/forward_visitor.hpp new file mode 100644 index 0000000000..ac825f11ee --- /dev/null +++ b/src/mlpack/methods/ann/visitor/forward_visitor.hpp @@ -0,0 +1,54 @@ +/** + * @file methods/ann/visitor/forward_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Forward() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_FORWARD_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_FORWARD_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * ForwardVisitor executes the Forward() function given the input and output + * parameter. + */ +class ForwardVisitor : public boost::static_visitor +{ + public: + //! Execute the Forward() function given the input and output parameter. + ForwardVisitor(const arma::mat& input, arma::mat& output); + + //! Execute the Forward() function. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The input parameter set. + const arma::mat& input; + + //! The output parameter set. + arma::mat& output; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "forward_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp new file mode 100644 index 0000000000..248d852c53 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp @@ -0,0 +1,43 @@ +/** + * @file methods/ann/visitor/forward_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Forward() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_FORWARD_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_FORWARD_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "forward_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! ForwardVisitor visitor class. +inline ForwardVisitor::ForwardVisitor(const arma::mat& input, arma::mat& output) : + input(input), + output(output) +{ + /* Nothing to do here. */ +} + +template +inline void ForwardVisitor::operator()(LayerType* layer) const +{ + layer->Forward(input, output); +} + +inline void ForwardVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/gradient_set_visitor.hpp b/src/mlpack/methods/ann/visitor/gradient_set_visitor.hpp new file mode 100644 index 0000000000..4c492b3f0e --- /dev/null +++ b/src/mlpack/methods/ann/visitor/gradient_set_visitor.hpp @@ -0,0 +1,82 @@ +/** + * @file methods/ann/visitor/gradient_set_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Gradient() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_GRADIENT_SET_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_GRADIENT_SET_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * GradientSetVisitor update the gradient parameter given the gradient set. + */ +class GradientSetVisitor : public boost::static_visitor +{ + public: + //! Update the gradient parameter given the gradient set. + GradientSetVisitor(arma::mat& gradient, size_t offset = 0); + + //! Update the gradient parameter. + template + size_t operator()(LayerType* layer) const; + + size_t operator()(MoreTypes layer) const; + + private: + //! The gradient set. + arma::mat& gradient; + + //! The gradient offset. + size_t offset; + + //! Update the gradient if the module implements the Gradient() function. + template + typename std::enable_if< + HasGradientCheck::value && + !HasModelCheck::value, size_t>::type + LayerGradients(T* layer, arma::mat& input) const; + + //! Update the gradient if the module implements the Model() function. + template + typename std::enable_if< + !HasGradientCheck::value && + HasModelCheck::value, size_t>::type + LayerGradients(T* layer, arma::mat& input) const; + + //! Update the gradient if the module implements the Gradient() and Model() + //! function. + template + typename std::enable_if< + HasGradientCheck::value && + HasModelCheck::value, size_t>::type + LayerGradients(T* layer, arma::mat& input) const; + + //! Do not update the gradient parameter if the module doesn't implement the + //! Gradient() or Model() function. + template + typename std::enable_if< + !HasGradientCheck::value && + !HasModelCheck::value, size_t>::type + LayerGradients(T* layer, P& input) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "gradient_set_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/gradient_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/gradient_set_visitor_impl.hpp new file mode 100644 index 0000000000..85e148da25 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/gradient_set_visitor_impl.hpp @@ -0,0 +1,100 @@ +/** + * @file methods/ann/visitor/gradient_set_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Gradient() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_GRADIENT_SET_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_GRADIENT_SET_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "gradient_set_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! GradientSetVisitor visitor class. +inline GradientSetVisitor::GradientSetVisitor(arma::mat& gradient, + size_t offset) : + gradient(gradient), + offset(offset) +{ + /* Nothing to do here. */ +} + +template +inline size_t GradientSetVisitor::operator()(LayerType* layer) const +{ + return LayerGradients(layer, layer->OutputParameter()); +} + +inline size_t GradientSetVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + HasGradientCheck::value && + !HasModelCheck::value, size_t>::type +GradientSetVisitor::LayerGradients(T* layer, arma::mat& /* input */) const +{ + layer->Gradient() = arma::mat(gradient.memptr() + offset, + layer->Parameters().n_rows, layer->Parameters().n_cols, false, false); + + return layer->Parameters().n_elem; +} + +template +inline typename std::enable_if< + !HasGradientCheck::value && + HasModelCheck::value, size_t>::type +GradientSetVisitor::LayerGradients(T* layer, arma::mat& /* input */) const +{ + size_t modelOffset = 0; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(GradientSetVisitor( + gradient, modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +template +inline typename std::enable_if< + HasGradientCheck::value && + HasModelCheck::value, size_t>::type +GradientSetVisitor::LayerGradients(T* layer, arma::mat& /* input */) const +{ + layer->Gradient() = arma::mat(gradient.memptr() + offset, + layer->Parameters().n_rows, layer->Parameters().n_cols, false, false); + + size_t modelOffset = layer->Parameters().n_elem; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(GradientSetVisitor( + gradient, modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +template +inline typename std::enable_if< + !HasGradientCheck::value && + !HasModelCheck::value, size_t>::type +GradientSetVisitor::LayerGradients(T* /* layer */, P& /* input */) const +{ + return 0; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/gradient_update_visitor.hpp b/src/mlpack/methods/ann/visitor/gradient_update_visitor.hpp new file mode 100644 index 0000000000..feedf0d299 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/gradient_update_visitor.hpp @@ -0,0 +1,82 @@ +/** + * @file methods/ann/visitor/gradient_update_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Gradient() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_GRADIENT_UPDATE_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_GRADIENT_UPDATE_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * GradientUpdateVisitor update the gradient parameter given the gradient set. + */ +class GradientUpdateVisitor : public boost::static_visitor +{ + public: + //! Update the gradient parameter given the gradient set. + GradientUpdateVisitor(arma::mat& gradient, size_t offset = 0); + + //! Update the gradient parameter. + template + size_t operator()(LayerType* layer) const; + + size_t operator()(MoreTypes layer) const; + + private: + //! The gradient set. + arma::mat& gradient; + + //! The gradient offset. + size_t offset; + + //! Update the gradient if the module implements the Gradient() function. + template + typename std::enable_if< + HasGradientCheck::value && + !HasModelCheck::value, size_t>::type + LayerGradients(T* layer, arma::mat& input) const; + + //! Update the gradient if the module implements the Model() function. + template + typename std::enable_if< + !HasGradientCheck::value && + HasModelCheck::value, size_t>::type + LayerGradients(T* layer, arma::mat& input) const; + + //! Update the gradient if the module implements the Gradient() and Model() + //! function. + template + typename std::enable_if< + HasGradientCheck::value && + HasModelCheck::value, size_t>::type + LayerGradients(T* layer, arma::mat& input) const; + + //! Do not update the gradient parameter if the module doesn't implement the + //! Gradient() or Model() function. + template + typename std::enable_if< + !HasGradientCheck::value && + !HasModelCheck::value, size_t>::type + LayerGradients(T* layer, P& input) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "gradient_update_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/gradient_update_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/gradient_update_visitor_impl.hpp new file mode 100644 index 0000000000..f233eaf65d --- /dev/null +++ b/src/mlpack/methods/ann/visitor/gradient_update_visitor_impl.hpp @@ -0,0 +1,106 @@ +/** + * @file methods/ann/visitor/gradient_update_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Gradient() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_GRADIENT_UPDATE_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_GRADIENT_UPDATE_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "gradient_update_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! GradientUpdateVisitor visitor class. +inline GradientUpdateVisitor::GradientUpdateVisitor(arma::mat& gradient, + size_t offset) : + gradient(gradient), + offset(offset) +{ + /* Nothing to do here. */ +} + +template +inline size_t GradientUpdateVisitor::operator()(LayerType* layer) const +{ + return LayerGradients(layer, layer->OutputParameter()); +} + +inline size_t GradientUpdateVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + HasGradientCheck::value && + !HasModelCheck::value, size_t>::type +GradientUpdateVisitor::LayerGradients(T* layer, arma::mat& /* input */) const +{ + if (layer->Parameters().n_elem != 0) + { + layer->Gradient() = gradient.submat(offset, 0, + offset + layer->Parameters().n_elem - 1, 0);; + } + + return layer->Parameters().n_elem; +} + +template +inline typename std::enable_if< + !HasGradientCheck::value && + HasModelCheck::value, size_t>::type +GradientUpdateVisitor::LayerGradients(T* layer, arma::mat& /* input */) const +{ + size_t modelOffset = 0; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(GradientUpdateVisitor( + gradient, modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +template +inline typename std::enable_if< + HasGradientCheck::value && + HasModelCheck::value, size_t>::type +GradientUpdateVisitor::LayerGradients(T* layer, arma::mat& /* input */) const +{ + if (layer->Parameters().n_elem != 0) + { + layer->Gradient() = gradient.submat(offset, 0, + offset + layer->Parameters().n_elem - 1, 0);; + } + + size_t modelOffset = layer->Parameters().n_elem; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(GradientUpdateVisitor( + gradient, modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +template +inline typename std::enable_if< + !HasGradientCheck::value && + !HasModelCheck::value, size_t>::type +GradientUpdateVisitor::LayerGradients(T* /* layer */, P& /* input */) const +{ + return 0; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/gradient_visitor.hpp b/src/mlpack/methods/ann/visitor/gradient_visitor.hpp new file mode 100644 index 0000000000..fc04c96161 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/gradient_visitor.hpp @@ -0,0 +1,89 @@ +/** + * @file methods/ann/visitor/gradient_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Gradient() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_GRADIENT_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_GRADIENT_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * SearchModeVisitor executes the Gradient() method of the given module using + * the input and delta parameter. + */ +class GradientVisitor : public boost::static_visitor +{ + public: + //! Executes the Gradient() method of the given module using the input and + //! delta parameter. + GradientVisitor(const arma::mat& input, const arma::mat& delta); + + //! Executes the Gradient() method for the layer with the specified index. + GradientVisitor(const arma::mat& input, + const arma::mat& delta, + const size_t index); + + //! Executes the Gradient() method. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The input set. + const arma::mat& input; + + //! The delta parameter. + const arma::mat& delta; + + //! Index of the layer to run. + size_t index; + + //! Indicates whether to use index or not + bool hasIndex; + + //! Execute the Gradient() function if the module implements the Gradient() + //! function. + template + typename std::enable_if< + HasGradientCheck::value && + !HasRunCheck::value, void>::type + LayerGradients(T* layer, arma::mat& input) const; + + //! Execute the Gradient() function if the module implements the Gradient() + //! and has a Run() function. + template + typename std::enable_if< + HasGradientCheck::value && + HasRunCheck::value, void>::type + LayerGradients(T* layer, arma::mat& input) const; + + //! Do not execute the Gradient() function if the module doesn't implement + //! the Gradient() function. + template + typename std::enable_if< + !HasGradientCheck::value, void>::type + LayerGradients(T* layer, P& input) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "gradient_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/gradient_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/gradient_visitor_impl.hpp new file mode 100644 index 0000000000..3537aa9959 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/gradient_visitor_impl.hpp @@ -0,0 +1,90 @@ +/** + * @file methods/ann/visitor/gradient_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Gradient() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_GRADIENT_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_GRADIENT_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "gradient_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! GradientVisitor visitor class. +inline GradientVisitor::GradientVisitor(const arma::mat& input, + const arma::mat& delta) : + input(input), + delta(delta), + index(0), + hasIndex(false) +{ + /* Nothing to do here. */ +} + +inline GradientVisitor::GradientVisitor(const arma::mat& input, + const arma::mat& delta, + const size_t index) : + input(input), + delta(delta), + index(index), + hasIndex(true) +{ + /* Nothing to do here. */ +} + +template +inline void GradientVisitor::operator()(LayerType* layer) const +{ + LayerGradients(layer, layer->OutputParameter()); +} + +inline void GradientVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + HasGradientCheck::value && + !HasRunCheck::value, void>::type +GradientVisitor::LayerGradients(T* layer, arma::mat& /* input */) const +{ + layer->Gradient(input, delta, layer->Gradient()); +} + +template +inline typename std::enable_if< + HasGradientCheck::value && + HasRunCheck::value, void>::type +GradientVisitor::LayerGradients(T* layer, arma::mat& /* input */) const +{ + if (!hasIndex) + { + layer->Gradient(input, delta, layer->Gradient()); + } + else + { + layer->Gradient(input, delta, layer->Gradient(), index); + } +} + +template +inline typename std::enable_if< + !HasGradientCheck::value, void>::type +GradientVisitor::LayerGradients(T* /* layer */, P& /* input */) const +{ + /* Nothing to do here. */ +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/gradient_zero_visitor.hpp b/src/mlpack/methods/ann/visitor/gradient_zero_visitor.hpp new file mode 100644 index 0000000000..3480796089 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/gradient_zero_visitor.hpp @@ -0,0 +1,60 @@ +/** + * @file methods/ann/visitor/gradient_zero_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Gradient() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_GRADIENT_ZERO_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_GRADIENT_ZERO_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/* + * GradientZeroVisitor set the gradient to zero for the given module. + */ +class GradientZeroVisitor : public boost::static_visitor +{ + public: + //! Set the gradient to zero for the given module. + GradientZeroVisitor(); + + //! Set the gradient to zero. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! Set the gradient to zero if the module implements the Gradient() function. + template + typename std::enable_if< + HasGradientCheck::value, void>::type + LayerGradients(T* layer, arma::mat& input) const; + + //! Do not set the gradient to zero if the module doesn't implement the + //! Gradient() function. + template + typename std::enable_if< + !HasGradientCheck::value, void>::type + LayerGradients(T* layer, P& input) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "gradient_zero_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/gradient_zero_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/gradient_zero_visitor_impl.hpp new file mode 100644 index 0000000000..de39a692a6 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/gradient_zero_visitor_impl.hpp @@ -0,0 +1,57 @@ +/** + * @file methods/ann/visitor/gradient_zero_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Gradient() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_GRADIENT_ZERO_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_GRADIENT_ZERO_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "gradient_zero_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! GradientZeroVisitor visitor class. +inline GradientZeroVisitor::GradientZeroVisitor() +{ + /* Nothing to do here. */ +} + +template +inline void GradientZeroVisitor::operator()(LayerType* layer) const +{ + LayerGradients(layer, layer->OutputParameter()); +} + +inline void GradientZeroVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + HasGradientCheck::value, void>::type +GradientZeroVisitor::LayerGradients(T* layer, arma::mat& /* input */) const +{ + layer->Gradient().zeros(); +} + +template +inline typename std::enable_if< + !HasGradientCheck::value, void>::type +GradientZeroVisitor::LayerGradients(T* /* layer */, P& /* input */) const +{ + /* Nothing to do here. */ +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/input_shape_visitor.hpp b/src/mlpack/methods/ann/visitor/input_shape_visitor.hpp new file mode 100644 index 0000000000..c27135aae3 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/input_shape_visitor.hpp @@ -0,0 +1,58 @@ +/** + * @file methods/ann/visitor/input_shape_visitor.hpp + * @author Khizir Siddiqui + * @author Nippun Sharma + * + * This file provides an abstraction for the InputShape() function for + * different layers and automatically directs any parameter to the right layer + * type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_INPUT_SHAPE_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_INPUT_SHAPE_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * InShapeVisitor returns the input shape a Layer expects. + */ +class InShapeVisitor : public boost::static_visitor +{ + public: + //! Return the input shape of layer. + template + size_t operator()(LayerType* layer) const; + + size_t operator()(MoreTypes layer) const; + + private: + //! If the module doesn't implement the InputShape() function return 0. + template + typename std::enable_if< + !HasInputShapeCheck::value, size_t>::type + LayerInputShape(T* layer) const; + + //! If the module implements the InputShape() function returns the input shape. + template + typename std::enable_if< + HasInputShapeCheck::value, size_t>::type + LayerInputShape(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "input_shape_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/input_shape_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/input_shape_visitor_impl.hpp new file mode 100644 index 0000000000..bda5f7b604 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/input_shape_visitor_impl.hpp @@ -0,0 +1,53 @@ +/** + * @file methods/ann/visitor/input_shape_visitor_impl.hpp + * @author Khizir Siddiqui + * @author Nippun Sharma + * + * Implementation of the InputShape() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_INPUT_SHAPE_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_INPUT_SHAPE_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "input_shape_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! InShapeVisitor visitor class. +template +inline std::size_t InShapeVisitor::operator()(LayerType* layer) const +{ + return LayerInputShape(layer); +} + +inline std::size_t InShapeVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasInputShapeCheck::value, std::size_t>::type +InShapeVisitor::LayerInputShape(T* /* layer */) const +{ + return 0; +} + +template +inline typename std::enable_if< + HasInputShapeCheck::value, std::size_t>::type +InShapeVisitor::LayerInputShape(T* layer) const +{ + return layer->InputShape(); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/load_output_parameter_visitor.hpp b/src/mlpack/methods/ann/visitor/load_output_parameter_visitor.hpp new file mode 100644 index 0000000000..2546d52f90 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/load_output_parameter_visitor.hpp @@ -0,0 +1,65 @@ +/** + * @file methods/ann/visitor/load_output_parameter_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the OutputParameter() function for + * different layers and automatically directs any parameter to the right layer + * type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_LOAD_OUTPUT_PARAMETER_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_LOAD_OUTPUT_PARAMETER_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * LoadOutputParameterVisitor restores the output parameter using the given + * parameter set. + */ +class LoadOutputParameterVisitor : public boost::static_visitor +{ + public: + //! Restore the output parameter given a parameter set. + LoadOutputParameterVisitor(std::vector& parameter); + + //! Restore the output parameter. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The parameter set. + std::vector& parameter; + + //! Restore the output parameter for a module which doesn't implement the + //! Model() function. + template + typename std::enable_if< + !HasModelCheck::value, void>::type + OutputParameter(T* layer) const; + + //! Restore the output parameter for a module which implements the Model() + //! function. + template + typename std::enable_if< + HasModelCheck::value, void>::type + OutputParameter(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "load_output_parameter_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/load_output_parameter_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/load_output_parameter_visitor_impl.hpp new file mode 100644 index 0000000000..5c384643ee --- /dev/null +++ b/src/mlpack/methods/ann/visitor/load_output_parameter_visitor_impl.hpp @@ -0,0 +1,66 @@ +/** + * @file methods/ann/visitor/load_output_parameter_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the OutputParameter() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_LOAD_OUTPUT_PARAMETER_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_LOAD_OUTPUT_PARAMETER_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "load_output_parameter_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! LoadOutputParameterVisitor visitor class. +inline LoadOutputParameterVisitor::LoadOutputParameterVisitor( + std::vector& parameter) : parameter(parameter) +{ + /* Nothing to do here. */ +} + +template +inline void LoadOutputParameterVisitor::operator()(LayerType* layer) const +{ + OutputParameter(layer); +} + +inline void LoadOutputParameterVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasModelCheck::value, void>::type +LoadOutputParameterVisitor::OutputParameter(T* layer) const +{ + layer->OutputParameter() = parameter.back(); + parameter.pop_back(); +} + +template +inline typename std::enable_if< + HasModelCheck::value, void>::type +LoadOutputParameterVisitor::OutputParameter(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(LoadOutputParameterVisitor(parameter), + layer->Model()[layer->Model().size() - i - 1]); + } + + layer->OutputParameter() = parameter.back(); + parameter.pop_back(); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/loss_visitor.hpp b/src/mlpack/methods/ann/visitor/loss_visitor.hpp new file mode 100644 index 0000000000..d9ca99763f --- /dev/null +++ b/src/mlpack/methods/ann/visitor/loss_visitor.hpp @@ -0,0 +1,71 @@ +/** + * @file methods/ann/visitor/loss_visitor.hpp + * @author Atharva Khandait + * + * This file provides an abstraction for the Loss() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * LossVisitor exposes the Loss() method of the given module. + */ +class LossVisitor : public boost::static_visitor +{ + public: + //! Return the Loss. + template + double operator()(LayerType* layer) const; + + double operator()(MoreTypes layer) const; + + private: + //! Return 0 if the module doesn't implement the Loss() or Model() function. + template + typename std::enable_if< + !HasLoss::value && + !HasModelCheck::value, double>::type + LayerLoss(T* layer) const; + + //! Return the output height if the module implements the Loss() function. + template + typename std::enable_if< + HasLoss::value && + !HasModelCheck::value, double>::type + LayerLoss(T* layer) const; + + //! Return the loss if the module implements the Model() function. + template + typename std::enable_if< + !HasLoss::value && + HasModelCheck::value, double>::type + LayerLoss(T* layer) const; + + //! Return the loss if the module implements the Model() or loss() function. + template + typename std::enable_if< + HasLoss::value && + HasModelCheck::value, double>::type + LayerLoss(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "loss_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp new file mode 100644 index 0000000000..ee8d6cb402 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp @@ -0,0 +1,99 @@ +/** + * @file methods/ann/visitor/loss_visitor_impl.hpp + * @author Atharva Khandait + * + * Implementation of the Loss() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "loss_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! LossVisitor visitor class. +template +inline double LossVisitor::operator()(LayerType* layer) const +{ + return LayerLoss(layer); +} + +inline double LossVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasLoss::value && + !HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* /* layer */) const +{ + return 0; +} + +template +inline typename std::enable_if< + HasLoss::value && + !HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* layer) const +{ + return layer->Loss(); +} + +template +inline typename std::enable_if< + !HasLoss::value && + HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + double loss = boost::apply_visitor(LossVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (loss != 0) + { + return loss; + } + } + + return 0; +} + +template +inline typename std::enable_if< + HasLoss::value && + HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* layer) const +{ + double loss = layer->Loss(); + + if (loss == 0) + { + for (size_t i = 0; i < layer->Model().size(); ++i) + { + loss = boost::apply_visitor(LossVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (loss != 0) + { + return loss; + } + } + } + + return loss; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/output_height_visitor.hpp b/src/mlpack/methods/ann/visitor/output_height_visitor.hpp new file mode 100644 index 0000000000..f14350b9ef --- /dev/null +++ b/src/mlpack/methods/ann/visitor/output_height_visitor.hpp @@ -0,0 +1,75 @@ +/** + * @file methods/ann/visitor/output_height_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the OutputHeight() function for + * different layers and automatically directs any parameter to the right layer + * type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_OUTPUT_HEIGHT_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_OUTPUT_HEIGHT_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * OutputHeightVisitor exposes the OutputHeight() method of the given module. + */ +class OutputHeightVisitor : public boost::static_visitor +{ + public: + //! Return the output height. + template + size_t operator()(LayerType* layer) const; + + size_t operator()(MoreTypes layer) const; + + private: + //! Return 0 if the module doesn't implement the InputHeight() or Model() + //! function. + template + typename std::enable_if< + !HasInputHeight::value && + !HasModelCheck::value, size_t>::type + LayerOutputHeight(T* layer) const; + + //! Return the output height if the module implements the InputHeight() + //! function. + template + typename std::enable_if< + HasInputHeight::value && + !HasModelCheck::value, size_t>::type + LayerOutputHeight(T* layer) const; + + //! Return the output height if the module implements the Model() function. + template + typename std::enable_if< + !HasInputHeight::value && + HasModelCheck::value, size_t>::type + LayerOutputHeight(T* layer) const; + + //! Return the output height if the module implements the Model() or + //! InputHeight() function. + template + typename std::enable_if< + HasInputHeight::value && + HasModelCheck::value, size_t>::type + LayerOutputHeight(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "output_height_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/output_height_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/output_height_visitor_impl.hpp new file mode 100644 index 0000000000..ae219da220 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/output_height_visitor_impl.hpp @@ -0,0 +1,99 @@ +/** + * @file methods/ann/visitor/output_height_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the OutputHeight() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_OUTPUT_HEIGHT_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_OUTPUT_HEIGHT_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "output_height_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! OutputHeightVisitor visitor class. +template +inline size_t OutputHeightVisitor::operator()(LayerType* layer) const +{ + return LayerOutputHeight(layer); +} + +inline size_t OutputHeightVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasInputHeight::value && + !HasModelCheck::value, size_t>::type +OutputHeightVisitor::LayerOutputHeight(T* /* layer */) const +{ + return 0; +} + +template +inline typename std::enable_if< + HasInputHeight::value && + !HasModelCheck::value, size_t>::type +OutputHeightVisitor::LayerOutputHeight(T* layer) const +{ + return layer->OutputHeight(); +} + +template +inline typename std::enable_if< + !HasInputHeight::value && + HasModelCheck::value, size_t>::type +OutputHeightVisitor::LayerOutputHeight(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + size_t outputHeight = boost::apply_visitor(OutputHeightVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (outputHeight != 0) + { + return outputHeight; + } + } + + return 0; +} + +template +inline typename std::enable_if< + HasInputHeight::value && + HasModelCheck::value, size_t>::type +OutputHeightVisitor::LayerOutputHeight(T* layer) const +{ + size_t outputHeight = layer->OutputHeight(); + + if (outputHeight == 0) + { + for (size_t i = 0; i < layer->Model().size(); ++i) + { + outputHeight = boost::apply_visitor(OutputHeightVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (outputHeight != 0) + { + return outputHeight; + } + } + } + + return outputHeight; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/output_parameter_visitor.hpp b/src/mlpack/methods/ann/visitor/output_parameter_visitor.hpp new file mode 100644 index 0000000000..6a12226464 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/output_parameter_visitor.hpp @@ -0,0 +1,43 @@ +/** + * @file methods/ann/visitor/output_parameter_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the OutputParameter() function for + * different layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_OUTPUT_PARAMETER_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_OUTPUT_PARAMETER_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * OutputParameterVisitor exposes the output parameter of the given module. + */ +class OutputParameterVisitor : public boost::static_visitor +{ + public: + //! Return the output parameter set. + template + arma::mat& operator()(LayerType* layer) const; + + arma::mat& operator()(MoreTypes layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "output_parameter_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/output_parameter_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/output_parameter_visitor_impl.hpp new file mode 100644 index 0000000000..5086669548 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/output_parameter_visitor_impl.hpp @@ -0,0 +1,36 @@ +/** + * @file methods/ann/visitor/output_parameter_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the OutputParameter() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_OUTPUT_PARAMETER_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_OUTPUT_PARAMETER_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "output_parameter_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! OutputParameterVisitor visitor class. +template +inline arma::mat& OutputParameterVisitor::operator()(LayerType *layer) const +{ + return layer->OutputParameter(); +} + +inline arma::mat& OutputParameterVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/output_width_visitor.hpp b/src/mlpack/methods/ann/visitor/output_width_visitor.hpp new file mode 100644 index 0000000000..d6a0fa83ed --- /dev/null +++ b/src/mlpack/methods/ann/visitor/output_width_visitor.hpp @@ -0,0 +1,75 @@ +/** + * @file methods/ann/visitor/output_width_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the OutputWidth() function for + * different layers and automatically directs any parameter to the right layer + * type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_OUTPUT_WIDTH_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_OUTPUT_WIDTH_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * OutputWidthVisitor exposes the OutputWidth() method of the given module. + */ +class OutputWidthVisitor : public boost::static_visitor +{ + public: + //! Return the output width. + template + size_t operator()(LayerType* layer) const; + + size_t operator()(MoreTypes layer) const; + + private: + //! Return 0 if the module doesn't implement the InputWidth() or Model() + //! function. + template + typename std::enable_if< + !HasInputWidth::value && + !HasModelCheck::value, size_t>::type + LayerOutputWidth(T* layer) const; + + //! Return the output width if the module implements the InputWidth() + //! function. + template + typename std::enable_if< + HasInputWidth::value && + !HasModelCheck::value, size_t>::type + LayerOutputWidth(T* layer) const; + + //! Return the output width if the module implements the Model() function. + template + typename std::enable_if< + !HasInputWidth::value && + HasModelCheck::value, size_t>::type + LayerOutputWidth(T* layer) const; + + //! Return the output width if the module implements the Model() or + //! InputWidth() function. + template + typename std::enable_if< + HasInputWidth::value && + HasModelCheck::value, size_t>::type + LayerOutputWidth(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "output_width_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/output_width_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/output_width_visitor_impl.hpp new file mode 100644 index 0000000000..3d68087d83 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/output_width_visitor_impl.hpp @@ -0,0 +1,99 @@ +/** + * @file methods/ann/visitor/output_width_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the OutputWidth() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_OUTPUT_WIDTH_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_OUTPUT_WIDTH_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "output_width_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! OutputWidthVisitor visitor class. +template +inline size_t OutputWidthVisitor::operator()(LayerType* layer) const +{ + return LayerOutputWidth(layer); +} + +inline size_t OutputWidthVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasInputWidth::value && + !HasModelCheck::value, size_t>::type +OutputWidthVisitor::LayerOutputWidth(T* /* layer */) const +{ + return 0; +} + +template +inline typename std::enable_if< + HasInputWidth::value && + !HasModelCheck::value, size_t>::type +OutputWidthVisitor::LayerOutputWidth(T* layer) const +{ + return layer->OutputWidth(); +} + +template +inline typename std::enable_if< + !HasInputWidth::value && + HasModelCheck::value, size_t>::type +OutputWidthVisitor::LayerOutputWidth(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + size_t outputWidth = boost::apply_visitor(OutputWidthVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (outputWidth != 0) + { + return outputWidth; + } + } + + return 0; +} + +template +inline typename std::enable_if< + HasInputWidth::value && + HasModelCheck::value, size_t>::type +OutputWidthVisitor::LayerOutputWidth(T* layer) const +{ + size_t outputWidth = layer->OutputWidth(); + + if (outputWidth == 0) + { + for (size_t i = 0; i < layer->Model().size(); ++i) + { + outputWidth = boost::apply_visitor(OutputWidthVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (outputWidth != 0) + { + return outputWidth; + } + } + } + + return outputWidth; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/parameters_set_visitor.hpp b/src/mlpack/methods/ann/visitor/parameters_set_visitor.hpp new file mode 100644 index 0000000000..c8cba59bb2 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/parameters_set_visitor.hpp @@ -0,0 +1,64 @@ +/** + * @file methods/ann/visitor/parameters_set_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Parameters() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_PARAMETERS_SET_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_PARAMETERS_SET_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * ParametersSetVisitor update the parameters set using the given matrix. + */ +class ParametersSetVisitor : public boost::static_visitor +{ + public: + //! Update the parameters set given the parameters matrix. + ParametersSetVisitor(arma::mat& parameters); + + //! Update the parameters set. + template + void operator()(LayerType *layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The parameters set. + arma::mat& parameters; + + //! Do not update the parameters set if the module doesn't implement the + //! Parameters() function. + template + typename std::enable_if< + !HasParametersCheck::value, void>::type + LayerParameters(T* layer, P& output) const; + + //! Update the parameters set if the module implements the Parameters() + //! function. + template + typename std::enable_if< + HasParametersCheck::value, void>::type + LayerParameters(T* layer, P& output) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "parameters_set_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/parameters_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/parameters_set_visitor_impl.hpp new file mode 100644 index 0000000000..820a5ab464 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/parameters_set_visitor_impl.hpp @@ -0,0 +1,58 @@ +/** + * @file methods/ann/visitor/parameters_set_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Parameters() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_PARAMETERS_SET_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_PARAMETERS_SET_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "parameters_set_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! ParametersSetVisitor visitor class. +inline ParametersSetVisitor::ParametersSetVisitor(arma::mat& parameters) : + parameters(parameters) +{ + /* Nothing to do here. */ +} + +template +inline void ParametersSetVisitor::operator()(LayerType *layer) const +{ + LayerParameters(layer, layer->OutputParameter()); +} + +inline void ParametersSetVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasParametersCheck::value, void>::type +ParametersSetVisitor::LayerParameters(T* /* layer */, P& /* output */) const +{ + /* Nothing to do here. */ +} + +template +inline typename std::enable_if< + HasParametersCheck::value, void>::type +ParametersSetVisitor::LayerParameters(T* layer, P& /* output */) const +{ + layer->Parameters() = parameters; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/parameters_visitor.hpp b/src/mlpack/methods/ann/visitor/parameters_visitor.hpp new file mode 100644 index 0000000000..36a0b30bf5 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/parameters_visitor.hpp @@ -0,0 +1,64 @@ +/** + * @file methods/ann/visitor/parameters_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Parameters() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_PARAMETERS_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_PARAMETERS_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * ParametersVisitor exposes the parameters set of the given module and stores + * the parameters set into the given matrix. + */ +class ParametersVisitor : public boost::static_visitor +{ + public: + //! Store the parameters set into the given parameters matrix. + ParametersVisitor(arma::mat& parameters); + + //! Set the parameters set. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The parameters set. + arma::mat& parameters; + + //! Do not set the parameters set if the module doesn't implement the + //! Parameters() function. + template + typename std::enable_if< + !HasParametersCheck::value, void>::type + LayerParameters(T* layer, P& output) const; + + //! Set the parameters set if the module implements the Parameters() function. + template + typename std::enable_if< + HasParametersCheck::value, void>::type + LayerParameters(T* layer, P& output) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "parameters_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/parameters_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/parameters_visitor_impl.hpp new file mode 100644 index 0000000000..c58604c995 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/parameters_visitor_impl.hpp @@ -0,0 +1,58 @@ +/** + * @file methods/ann/visitor/parameters_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Parameters() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_PARAMETERS_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_PARAMETERS_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "parameters_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! ParametersVisitor visitor class. +inline ParametersVisitor::ParametersVisitor(arma::mat& parameters) : + parameters(parameters) +{ + /* Nothing to do here. */ +} + +template +inline void ParametersVisitor::operator()(LayerType *layer) const +{ + LayerParameters(layer, layer->OutputParameter()); +} + +inline void ParametersVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasParametersCheck::value, void>::type +ParametersVisitor::LayerParameters(T* /* layer */, P& /* output */) const +{ + /* Nothing to do here. */ +} + +template +inline typename std::enable_if< + HasParametersCheck::value, void>::type +ParametersVisitor::LayerParameters(T* layer, P& /* output */) const +{ + parameters = layer->Parameters(); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/reset_cell_visitor.hpp b/src/mlpack/methods/ann/visitor/reset_cell_visitor.hpp new file mode 100644 index 0000000000..ba88ce161b --- /dev/null +++ b/src/mlpack/methods/ann/visitor/reset_cell_visitor.hpp @@ -0,0 +1,62 @@ +/** + * @file methods/ann/visitor/reset_cell_visitor.hpp + * @author Sumedh Ghaisas + * + * Boost static visitor abstraction for calling ResetCell function on RNN cells. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_RESET_CELL_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_RESET_CELL_VISITOR_HPP + +#include +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * ResetCellVisitor executes the ResetCell() function. + */ +class ResetCellVisitor : public boost::static_visitor +{ + public: + //! Reset the cell using the given size. + ResetCellVisitor(const size_t size); + + //! Execute the ResetCell() function. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + size_t size; + + //! Execute the ResetCell() function for a module which implements + //! the ResetCell() function. + template + typename std::enable_if< + HasResetCellCheck::value, void>::type + ResetCell(T* layer) const; + + //! Do not execute the Reset() function for a module which doesn't implement + // the Reset() or Model() function. + template + typename std::enable_if< + !HasResetCellCheck::value, void>::type + ResetCell(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "reset_cell_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/reset_cell_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/reset_cell_visitor_impl.hpp new file mode 100644 index 0000000000..c687a553c0 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/reset_cell_visitor_impl.hpp @@ -0,0 +1,58 @@ +/** + * @file methods/ann/visitor/reset_cell_visitor_impl.hpp + * @author Sumedh Ghaisas + * + * Implementation of the ResetCell() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_RESET_CELL_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_RESET_CELL_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "reset_cell_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! ResetVisitor visitor class. +inline ResetCellVisitor::ResetCellVisitor(const size_t size) : size(size) +{ + /* Nothing to do here. */ +} + +//! ResetVisitor visitor class. +template +inline void ResetCellVisitor::operator()(LayerType* layer) const +{ + ResetCell(layer); +} + +inline void ResetCellVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + HasResetCellCheck::value, void>::type +ResetCellVisitor::ResetCell(T* layer) const +{ + layer->ResetCell(size); +} + +template +inline typename std::enable_if< + !HasResetCellCheck::value, void>::type +ResetCellVisitor::ResetCell(T* /* layer */) const +{ + /* Nothing to do here. */ +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/reset_visitor.hpp b/src/mlpack/methods/ann/visitor/reset_visitor.hpp new file mode 100644 index 0000000000..72545cf53d --- /dev/null +++ b/src/mlpack/methods/ann/visitor/reset_visitor.hpp @@ -0,0 +1,75 @@ +/** + * @file methods/ann/visitor/reset_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Reset() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_RESET_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_RESET_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * ResetVisitor executes the Reset() function. + */ +class ResetVisitor : public boost::static_visitor +{ + public: + //! Execute the Reset() function. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! Execute the Reset() function for a module which implements the Reset() + //! function. + template + typename std::enable_if< + HasResetCheck::value && + !HasModelCheck::value, void>::type + ResetParameter(T* layer) const; + + //! Execute the Reset() function for a module which implements the Model() + //! function. + template + typename std::enable_if< + !HasResetCheck::value && + HasModelCheck::value, void>::type + ResetParameter(T* layer) const; + + //! Execute the Reset() function for a module which implements the Reset() + //! and Model() function. + template + typename std::enable_if< + HasResetCheck::value && + HasModelCheck::value, void>::type + ResetParameter(T* layer) const; + + //! Do not execute the Reset() function for a module which doesn't implement + // the Reset() or Model() function. + template + typename std::enable_if< + !HasResetCheck::value && + !HasModelCheck::value, void>::type + ResetParameter(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "reset_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/reset_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/reset_visitor_impl.hpp new file mode 100644 index 0000000000..9754c9baa5 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/reset_visitor_impl.hpp @@ -0,0 +1,80 @@ +/** + * @file methods/ann/visitor/reset_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Reset() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_RESET_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_RESET_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "reset_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! ResetVisitor visitor class. +template +inline void ResetVisitor::operator()(LayerType* layer) const +{ + ResetParameter(layer); +} + +inline void ResetVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + HasResetCheck::value && + !HasModelCheck::value, void>::type +ResetVisitor::ResetParameter(T* layer) const +{ + layer->Reset(); +} + +template +inline typename std::enable_if< + !HasResetCheck::value && + HasModelCheck::value, void>::type +ResetVisitor::ResetParameter(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(ResetVisitor(), layer->Model()[i]); + } +} + +template +inline typename std::enable_if< + HasResetCheck::value && + HasModelCheck::value, void>::type +ResetVisitor::ResetParameter(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(ResetVisitor(), layer->Model()[i]); + } + + layer->Reset(); +} + +template +inline typename std::enable_if< + !HasResetCheck::value && + !HasModelCheck::value, void>::type +ResetVisitor::ResetParameter(T* /* layer */) const +{ + /* Nothing to do here. */ +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/reward_set_visitor.hpp b/src/mlpack/methods/ann/visitor/reward_set_visitor.hpp new file mode 100644 index 0000000000..a4c6301d00 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/reward_set_visitor.hpp @@ -0,0 +1,81 @@ +/** + * @file methods/ann/visitor/reward_set_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Reward() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_REWARD_SET_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_REWARD_SET_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * RewardSetVisitor set the reward parameter given the reward value. + */ +class RewardSetVisitor : public boost::static_visitor +{ + public: + //! Set the reward parameter given the reward value. + RewardSetVisitor(const double reward); + + //! Set the reward parameter. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The reward value. + const double reward; + + //! Set the deterministic parameter if the module implements the + //! Deterministic() and Model() function. + template + typename std::enable_if< + HasRewardCheck::value && + HasModelCheck::value, void>::type + LayerReward(T* layer) const; + + //! Set the deterministic parameter if the module implements the + //! Model() function. + template + typename std::enable_if< + !HasRewardCheck::value && + HasModelCheck::value, void>::type + LayerReward(T* layer) const; + + //! Set the deterministic parameter if the module implements the + //! Deterministic() function. + template + typename std::enable_if< + HasRewardCheck::value && + !HasModelCheck::value, void>::type + LayerReward(T* layer) const; + + //! Do not set the deterministic parameter if the module doesn't implement the + //! Deterministic() or Model() function. + template + typename std::enable_if< + !HasRewardCheck::value && + !HasModelCheck::value, void>::type + LayerReward(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "reward_set_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/reward_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/reward_set_visitor_impl.hpp new file mode 100644 index 0000000000..8bc0eb5a21 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/reward_set_visitor_impl.hpp @@ -0,0 +1,87 @@ +/** + * @file methods/ann/visitor/reward_set_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Reward() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_REWARD_SET_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_REWARD_SET_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "reward_set_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! RewardSetVisitor visitor class. +inline RewardSetVisitor::RewardSetVisitor(const double reward) : reward(reward) +{ + /* Nothing to do here. */ +} + +template +inline void RewardSetVisitor::operator()(LayerType* layer) const +{ + LayerReward(layer); +} + +inline void RewardSetVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + HasRewardCheck::value && + HasModelCheck::value, void>::type +RewardSetVisitor::LayerReward(T* layer) const +{ + layer->Reward() = reward; + + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(RewardSetVisitor(reward), + layer->Model()[i]); + } +} + +template +inline typename std::enable_if< + !HasRewardCheck::value && + HasModelCheck::value, void>::type +RewardSetVisitor::LayerReward(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(RewardSetVisitor(reward), + layer->Model()[i]); + } +} + +template +inline typename std::enable_if< + HasRewardCheck::value && + !HasModelCheck::value, void>::type +RewardSetVisitor::LayerReward(T* layer) const +{ + layer->Reward() = reward; +} + +template +inline typename std::enable_if< + !HasRewardCheck::value && + !HasModelCheck::value, void>::type +RewardSetVisitor::LayerReward(T* /* input */) const +{ + /* Nothing to do here. */ +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/run_set_visitor.hpp b/src/mlpack/methods/ann/visitor/run_set_visitor.hpp new file mode 100644 index 0000000000..b993ea07f2 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/run_set_visitor.hpp @@ -0,0 +1,83 @@ +/** + * @file methods/ann/visitor/run_set_visitor.hpp + * @author Saksham Bansal + * + * This file provides an abstraction for the Run() function for + * different layers and automatically directs any parameter to the right layer + * type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_RUN_SET_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_RUN_SET_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * RunSetVisitor set the run parameter given the + * run value. + */ +class RunSetVisitor : public boost::static_visitor +{ + public: + //! Set the run parameter given the current run value. + RunSetVisitor(const bool run = true); + + //! Set the run parameter. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The run parameter. + const bool run; + + //! Set the run parameter if the module implements the + //! Run() and Model() function. + template + typename std::enable_if< + HasRunCheck::value && + HasModelCheck::value, void>::type + LayerRun(T* layer) const; + + //! Set the run parameter if the module implements the + //! Model() function. + template + typename std::enable_if< + !HasRunCheck::value && + HasModelCheck::value, void>::type + LayerRun(T* layer) const; + + //! Set the run parameter if the module implements the + //! Run() function. + template + typename std::enable_if< + HasRunCheck::value && + !HasModelCheck::value, void>::type + LayerRun(T* layer) const; + + //! Do not set the run parameter if the module doesn't implement the + //! Run() or Model() function. + template + typename std::enable_if< + !HasRunCheck::value && + !HasModelCheck::value, void>::type + LayerRun(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "run_set_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/run_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/run_set_visitor_impl.hpp new file mode 100644 index 0000000000..5e0ece0217 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/run_set_visitor_impl.hpp @@ -0,0 +1,88 @@ +/** + * @file methods/ann/visitor/run_set_visitor_impl.hpp + * @author Saksham Bansal + * + * Implementation of the Run() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_RUN_SET_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_RUN_SET_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "run_set_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! RunSetVisitor visitor class. +inline RunSetVisitor::RunSetVisitor( + const bool run) : run(run) +{ + /* Nothing to do here. */ +} + +template +inline void RunSetVisitor::operator()(LayerType* layer) const +{ + LayerRun(layer); +} + +inline void RunSetVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + HasRunCheck::value && + HasModelCheck::value, void>::type +RunSetVisitor::LayerRun(T* layer) const +{ + layer->Run() = run; + + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(RunSetVisitor(run), + layer->Model()[i]); + } +} + +template +inline typename std::enable_if< + !HasRunCheck::value && + HasModelCheck::value, void>::type +RunSetVisitor::LayerRun(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(RunSetVisitor(run), + layer->Model()[i]); + } +} + +template +inline typename std::enable_if< + HasRunCheck::value && + !HasModelCheck::value, void>::type +RunSetVisitor::LayerRun(T* layer) const +{ + layer->Run() = run; +} + +template +inline typename std::enable_if< + !HasRunCheck::value && + !HasModelCheck::value, void>::type +RunSetVisitor::LayerRun(T* /* input */) const +{ + /* Nothing to do here. */ +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/save_output_parameter_visitor.hpp b/src/mlpack/methods/ann/visitor/save_output_parameter_visitor.hpp new file mode 100644 index 0000000000..2ff0c9ea4e --- /dev/null +++ b/src/mlpack/methods/ann/visitor/save_output_parameter_visitor.hpp @@ -0,0 +1,64 @@ +/** + * @file methods/ann/visitor/save_output_parameter_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the OutputParameter() function for + * different layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_SAVE_OUTPUT_PARAMETER_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_SAVE_OUTPUT_PARAMETER_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * SaveOutputParameterVisitor saves the output parameter into the given + * parameter set. + */ +class SaveOutputParameterVisitor : public boost::static_visitor +{ + public: + //! Save the output parameter into the given parameter set. + SaveOutputParameterVisitor(std::vector& parameter); + + //! Save the output parameter. + template + void operator()(LayerType* layer) const; + + void operator()(MoreTypes layer) const; + + private: + //! The parameter set. + std::vector& parameter; + + //! Save the output parameter for a module which doesn't implement the + //! Model() function. + template + typename std::enable_if< + !HasModelCheck::value, void>::type + OutputParameter(T* layer) const; + + //! Save the output parameter for a module which implements the Model() + //! function. + template + typename std::enable_if< + HasModelCheck::value, void>::type + OutputParameter(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "save_output_parameter_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/save_output_parameter_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/save_output_parameter_visitor_impl.hpp new file mode 100644 index 0000000000..cc3559c165 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/save_output_parameter_visitor_impl.hpp @@ -0,0 +1,64 @@ +/** + * @file methods/ann/visitor/save_output_parameter_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the OutputParameter() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_SAVE_OUTPUT_PARAMETER_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_SAVE_OUTPUT_PARAMETER_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "load_output_parameter_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! SaveOutputParameterVisitor visitor class. +inline SaveOutputParameterVisitor::SaveOutputParameterVisitor( + std::vector& parameter) : parameter(parameter) +{ + /* Nothing to do here. */ +} + +template +inline void SaveOutputParameterVisitor::operator()(LayerType* layer) const +{ + OutputParameter(layer); +} + +inline void SaveOutputParameterVisitor::operator()(MoreTypes layer) const +{ + layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasModelCheck::value, void>::type +SaveOutputParameterVisitor::OutputParameter(T* layer) const +{ + parameter.push_back(layer->OutputParameter()); +} + +template +inline typename std::enable_if< + HasModelCheck::value, void>::type +SaveOutputParameterVisitor::OutputParameter(T* layer) const +{ + parameter.push_back(layer->OutputParameter()); + + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(SaveOutputParameterVisitor(parameter), + layer->Model()[i]); + } +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/set_input_height_visitor.hpp b/src/mlpack/methods/ann/visitor/set_input_height_visitor.hpp new file mode 100644 index 0000000000..7e47e4ca13 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/set_input_height_visitor.hpp @@ -0,0 +1,84 @@ +/** + * @file methods/ann/visitor/set_input_height_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the InputHeight() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_SET_INPUT_HEIGHT_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_SET_INPUT_HEIGHT_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * SetInputHeightVisitor updates the input height parameter with the given input + * height. + */ +class SetInputHeightVisitor : public boost::static_visitor +{ + public: + //! Update the input height parameter with the given input height. + SetInputHeightVisitor(const size_t inputHeight = 0, const bool reset = false); + + //! Update the input height parameter. + template + bool operator()(LayerType* layer) const; + + bool operator()(MoreTypes layer) const; + + private: + //! The input height parameter. + size_t inputHeight; + + //! If set reset the height parameter if already set. + bool reset; + + //! Do nothing if the module doesn't implement the InputHeight() or Model() + //! function. + template + typename std::enable_if< + !HasInputHeight::value && + !HasModelCheck::value, bool>::type + LayerInputHeight(T* layer) const; + + //! Update the input height if the module implements the InputHeight() + //! function. + template + typename std::enable_if< + HasInputHeight::value && + !HasModelCheck::value, bool>::type + LayerInputHeight(T* layer) const; + + //! Update the input height if the module implements the Model() function. + template + typename std::enable_if< + !HasInputHeight::value && + HasModelCheck::value, bool>::type + LayerInputHeight(T* layer) const; + + //! Update the input height if the module implements the InputHeight() or + //! Model() function. + template + typename std::enable_if< + HasInputHeight::value && + HasModelCheck::value, bool>::type + LayerInputHeight(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "set_input_height_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/set_input_height_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/set_input_height_visitor_impl.hpp new file mode 100644 index 0000000000..a5a3f8c0f7 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/set_input_height_visitor_impl.hpp @@ -0,0 +1,102 @@ +/** + * @file methods/ann/visitor/set_input_height_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the InputHeight() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_SET_INPUT_HEIGHT_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_SET_INPUT_HEIGHT_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "set_input_height_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! SetInputHeightVisitor visitor class. +inline SetInputHeightVisitor::SetInputHeightVisitor(const size_t inputHeight, + const bool reset) : + inputHeight(inputHeight), + reset(reset) +{ + /* Nothing to do here. */ +} + +template +inline bool SetInputHeightVisitor::operator()(LayerType* layer) const +{ + return LayerInputHeight(layer); +} + +inline bool SetInputHeightVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasInputHeight::value && + !HasModelCheck::value, bool>::type +SetInputHeightVisitor::LayerInputHeight(T* /* layer */) const +{ + return false; +} + +template +inline typename std::enable_if< + HasInputHeight::value && + !HasModelCheck::value, bool>::type +SetInputHeightVisitor::LayerInputHeight(T* layer) const +{ + if (layer->InputHeight() == 0 || reset) + { + layer->InputHeight() = inputHeight; + } + + return true; +} + +template +inline typename std::enable_if< + !HasInputHeight::value && + HasModelCheck::value, bool>::type +SetInputHeightVisitor::LayerInputHeight(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(SetInputHeightVisitor(inputHeight, reset), + layer->Model()[i]); + } + + return true; +} + +template +inline typename std::enable_if< + HasInputHeight::value && + HasModelCheck::value, bool>::type +SetInputHeightVisitor::LayerInputHeight(T* layer) const +{ + if (layer->InputHeight() == 0 || reset) + { + layer->InputHeight() = inputHeight; + } + + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(SetInputHeightVisitor(inputHeight, reset), + layer->Model()[i]); + } + + return true; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/set_input_width_visitor.hpp b/src/mlpack/methods/ann/visitor/set_input_width_visitor.hpp new file mode 100644 index 0000000000..f7fbd6f4e2 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/set_input_width_visitor.hpp @@ -0,0 +1,83 @@ +/** + * @file methods/ann/visitor/set_input_width_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the InputWidth() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_SET_INPUT_WIDTH_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_SET_INPUT_WIDTH_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * SetInputWidthVisitor updates the input width parameter with the given input + * width. + */ +class SetInputWidthVisitor : public boost::static_visitor +{ + public: + //! Update the input width parameter with the given input width. + SetInputWidthVisitor(const size_t inputWidth = 0, const bool reset = false); + + //! Update the input width parameter. + template + bool operator()(LayerType* layer) const; + + bool operator()(MoreTypes layer) const; + + private: + //! The input width parameter. + size_t inputWidth; + + //! If set reset the height parameter if already set. + bool reset; + + //! Do nothing if the module doesn't implement the InputWidth() or Model() + //! function. + template + typename std::enable_if< + !HasInputWidth::value && + !HasModelCheck::value, bool>::type + LayerInputWidth(T* layer) const; + + //! Update the input width if the module implements the InputWidth() function. + template + typename std::enable_if< + HasInputWidth::value && + !HasModelCheck::value, bool>::type + LayerInputWidth(T* layer) const; + + //! Update the input width if the module implements the Model() function. + template + typename std::enable_if< + !HasInputWidth::value && + HasModelCheck::value, bool>::type + LayerInputWidth(T* layer) const; + + //! Update the input width if the module implements the InputWidth() or + //! Model() function. + template + typename std::enable_if< + HasInputWidth::value && + HasModelCheck::value, bool>::type + LayerInputWidth(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "set_input_width_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/set_input_width_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/set_input_width_visitor_impl.hpp new file mode 100644 index 0000000000..56224ed009 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/set_input_width_visitor_impl.hpp @@ -0,0 +1,102 @@ +/** + * @file methods/ann/visitor/set_input_width_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the InputWidth() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_SET_INPUT_WIDTH_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_SET_INPUT_WIDTH_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "set_input_width_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! SetInputWidthVisitor visitor class. +inline SetInputWidthVisitor::SetInputWidthVisitor(const size_t inputWidth, + const bool reset) : + inputWidth(inputWidth), + reset(reset) +{ + /* Nothing to do here. */ +} + +template +inline bool SetInputWidthVisitor::operator()(LayerType* layer) const +{ + return LayerInputWidth(layer); +} + +inline bool SetInputWidthVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasInputWidth::value && + !HasModelCheck::value, bool>::type +SetInputWidthVisitor::LayerInputWidth(T* /* layer */) const +{ + return false; +} + +template +inline typename std::enable_if< + HasInputWidth::value && + !HasModelCheck::value, bool>::type +SetInputWidthVisitor::LayerInputWidth(T* layer) const +{ + if (layer->InputWidth() == 0 || reset) + { + layer->InputWidth() = inputWidth; + } + + return true; +} + +template +inline typename std::enable_if< + !HasInputWidth::value && + HasModelCheck::value, bool>::type +SetInputWidthVisitor::LayerInputWidth(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(SetInputWidthVisitor(inputWidth, reset), + layer->Model()[i]); + } + + return true; +} + +template +inline typename std::enable_if< + HasInputWidth::value && + HasModelCheck::value, bool>::type +SetInputWidthVisitor::LayerInputWidth(T* layer) const +{ + if (layer->InputWidth() == 0 || reset) + { + layer->InputWidth() = inputWidth; + } + + for (size_t i = 0; i < layer->Model().size(); ++i) + { + boost::apply_visitor(SetInputWidthVisitor(inputWidth, reset), + layer->Model()[i]); + } + + return true; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/weight_set_visitor.hpp b/src/mlpack/methods/ann/visitor/weight_set_visitor.hpp new file mode 100644 index 0000000000..81c6c110df --- /dev/null +++ b/src/mlpack/methods/ann/visitor/weight_set_visitor.hpp @@ -0,0 +1,82 @@ +/** + * @file methods/ann/visitor/weight_set_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the Weight() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_WEIGHT_SET_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_WEIGHT_SET_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * WeightSetVisitor update the module parameters given the parameters set. + */ +class WeightSetVisitor : public boost::static_visitor +{ + public: + //! Update the parameters given the parameters set and offset. + WeightSetVisitor(arma::mat& weight, const size_t offset = 0); + + //! Update the parameters set. + template + size_t operator()(LayerType* layer) const; + + size_t operator()(MoreTypes layer) const; + + private: + //! The parameters set. + arma::mat& weight; + + //! The parameters offset. + const size_t offset; + + //! Do not update the parameters if the module doesn't implement the + //! Parameters() or Model() function. + template + typename std::enable_if< + !HasParametersCheck::value && + !HasModelCheck::value, size_t>::type + LayerSize(T* layer, P&& input) const; + + //! Update the parameters if the module implements the Model() function. + template + typename std::enable_if< + !HasParametersCheck::value && + HasModelCheck::value, size_t>::type + LayerSize(T* layer, P&& input) const; + + //! Update the parameters if the module implements the Parameters() function. + template + typename std::enable_if< + HasParametersCheck::value && + !HasModelCheck::value, size_t>::type + LayerSize(T* layer, P&& input) const; + + //! Update the parameters if the module implements the Model() and + //! Parameters() function. + template + typename std::enable_if< + HasParametersCheck::value && + HasModelCheck::value, size_t>::type + LayerSize(T* layer, P&& input) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "weight_set_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/weight_set_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/weight_set_visitor_impl.hpp new file mode 100644 index 0000000000..fa52a469c7 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/weight_set_visitor_impl.hpp @@ -0,0 +1,100 @@ +/** + * @file methods/ann/visitor/weight_set_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the Weight() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_WEIGHT_SET_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_WEIGHT_SET_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "weight_set_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! WeightSetVisitor visitor class. +inline WeightSetVisitor::WeightSetVisitor(arma::mat& weight, + const size_t offset) : + weight(weight), + offset(offset) +{ + /* Nothing to do here. */ +} + +template +inline size_t WeightSetVisitor::operator()(LayerType* layer) const +{ + return LayerSize(layer, layer->OutputParameter()); +} + +inline size_t WeightSetVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasParametersCheck::value && + !HasModelCheck::value, size_t>::type +WeightSetVisitor::LayerSize(T* /* layer */, P&& /*output */) const +{ + return 0; +} + +template +inline typename std::enable_if< + !HasParametersCheck::value && + HasModelCheck::value, size_t>::type +WeightSetVisitor::LayerSize(T* layer, P&& /*output */) const +{ + size_t modelOffset = 0; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(WeightSetVisitor( + weight, modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +template +inline typename std::enable_if< + HasParametersCheck::value && + !HasModelCheck::value, size_t>::type +WeightSetVisitor::LayerSize(T* layer, P&& /* output */) const +{ + layer->Parameters() = arma::mat(weight.memptr() + offset, + layer->Parameters().n_rows, layer->Parameters().n_cols, false, false); + + return layer->Parameters().n_elem; +} + +template +inline typename std::enable_if< + HasParametersCheck::value && + HasModelCheck::value, size_t>::type +WeightSetVisitor::LayerSize(T* layer, P&& /* output */) const +{ + layer->Parameters() = arma::mat(weight.memptr() + offset, + layer->Parameters().n_rows, layer->Parameters().n_cols, false, false); + + size_t modelOffset = layer->Parameters().n_elem; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + modelOffset += boost::apply_visitor(WeightSetVisitor( + weight, modelOffset + offset), layer->Model()[i]); + } + + return modelOffset; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/weight_size_visitor.hpp b/src/mlpack/methods/ann/visitor/weight_size_visitor.hpp new file mode 100644 index 0000000000..074ca56614 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/weight_size_visitor.hpp @@ -0,0 +1,76 @@ +/** + * @file methods/ann/visitor/weight_size_visitor.hpp + * @author Marcus Edel + * + * This file provides an abstraction for the WeightSize() function for + * different layers and automatically directs any parameter to the right layer + * type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_WEIGHT_SIZE_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_WEIGHT_SIZE_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * WeightSizeVisitor returns the number of weights of the given module. + */ +class WeightSizeVisitor : public boost::static_visitor +{ + public: + //! Return the number of weights. + template + size_t operator()(LayerType* layer) const; + + size_t operator()(MoreTypes layer) const; + + private: + //! If the module doesn't implement the Parameters() or Model() function + //! return 0. + template + typename std::enable_if< + !HasParametersCheck::value && + !HasModelCheck::value, size_t>::type + LayerSize(T* layer, P& output) const; + + //! Return the number of parameters if the module implements the Model() + //! function. + template + typename std::enable_if< + !HasParametersCheck::value && + HasModelCheck::value, size_t>::type + LayerSize(T* layer, P& output) const; + + //! Return the number of parameters if the module implements the Parameters() + //! function. + template + typename std::enable_if< + HasParametersCheck::value && + !HasModelCheck::value, size_t>::type + LayerSize(T* layer, P& output) const; + + //! Return the accumulated number of parameters if the module implements the + //! Parameters() and Model() function. + template + typename std::enable_if< + HasParametersCheck::value && + HasModelCheck::value, size_t>::type + LayerSize(T* layer, P& output) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "weight_size_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/weight_size_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/weight_size_visitor_impl.hpp new file mode 100644 index 0000000000..50ef266a63 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/weight_size_visitor_impl.hpp @@ -0,0 +1,84 @@ +/** + * @file methods/ann/visitor/weight_size_visitor_impl.hpp + * @author Marcus Edel + * + * Implementation of the WeightSize() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_WEIGHT_SIZE_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_WEIGHT_SIZE_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "weight_size_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! WeightSizeVisitor visitor class. +template +inline size_t WeightSizeVisitor::operator()(LayerType* layer) const +{ + return LayerSize(layer, layer->OutputParameter()); +} + +inline size_t WeightSizeVisitor::operator()(MoreTypes layer) const +{ + return layer.apply_visitor(*this); +} + +template +inline typename std::enable_if< + !HasParametersCheck::value && + !HasModelCheck::value, size_t>::type +WeightSizeVisitor::LayerSize(T* /* layer */, P& /* output */) const +{ + return 0; +} + +template +inline typename std::enable_if< + !HasParametersCheck::value && + HasModelCheck::value, size_t>::type +WeightSizeVisitor::LayerSize(T* layer, P& /* output */) const +{ + size_t weights = 0; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + weights += boost::apply_visitor(WeightSizeVisitor(), layer->Model()[i]); + } + + return weights; +} + +template +inline typename std::enable_if< + HasParametersCheck::value && + !HasModelCheck::value, size_t>::type +WeightSizeVisitor::LayerSize(T* layer, P& /* output */) const +{ + return layer->Parameters().n_elem; +} + +template +inline typename std::enable_if< + HasParametersCheck::value && + HasModelCheck::value, size_t>::type +WeightSizeVisitor::LayerSize(T* layer, P& /* output */) const +{ + size_t weights = layer->Parameters().n_elem; + for (size_t i = 0; i < layer->Model().size(); ++i) + { + weights += boost::apply_visitor(WeightSizeVisitor(), layer->Model()[i]); + } + + return weights; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/kmeans/kmeans_impl.hpp b/src/mlpack/methods/kmeans/kmeans_impl.hpp index e8f782e916..4195da81ac 100644 --- a/src/mlpack/methods/kmeans/kmeans_impl.hpp +++ b/src/mlpack/methods/kmeans/kmeans_impl.hpp @@ -14,7 +14,6 @@ #include #include -#include namespace mlpack { namespace kmeans { @@ -162,8 +161,15 @@ Cluster(const MatType& data, // Check validity of initial guess. if (initialGuess) { - util::CheckSameSizes(centroids, clusters, "KMeans::Cluster()", "clusters"); - util::CheckSameDimensionality(data, centroids, "KMeans::Cluster()"); + if (centroids.n_cols != clusters) + Log::Fatal << "KMeans::Cluster(): wrong number of initial cluster " + << "centroids (" << centroids.n_cols << ", should be " << clusters + << ")!" << std::endl; + + if (centroids.n_rows != data.n_rows) + Log::Fatal << "KMeans::Cluster(): initial cluster centroids have wrong " + << " dimensionality (" << centroids.n_rows << ", should be " + << data.n_rows << ")!" << std::endl; } // Use the partitioner to come up with the partition assignments and calculate @@ -282,7 +288,10 @@ Cluster(const MatType& data, // Now, the initial assignments. First determine if they are necessary. if (initialAssignmentGuess) { - util::CheckSameSizes(data, assignments, "KMeans::Cluster()", "assignments"); + if (assignments.n_elem != data.n_cols) + Log::Fatal << "KMeans::Cluster(): initial cluster assignments (length " + << assignments.n_elem << ") not the same size as the dataset (size " + << data.n_cols << ")!" << std::endl; // Calculate initial centroids. arma::Row counts; diff --git a/src/mlpack/methods/linear_regression/linear_regression.cpp b/src/mlpack/methods/linear_regression/linear_regression.cpp index 774425ebc7..d3d4a2cd3f 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression.cpp @@ -12,7 +12,6 @@ */ #include "linear_regression.hpp" #include -#include using namespace mlpack; using namespace mlpack::regression; @@ -58,10 +57,6 @@ double LinearRegression::Train(const arma::mat& predictors, // We store the number of rows and columns of the predictors. // Reminder: Armadillo stores the data transposed from how we think of it, // that is, columns are actually rows (see: column major order). - - // Sanity check on data. - util::CheckSameSizes(predictors, responses, "LinearRegression::Train()"); - const size_t nCols = predictors.n_cols; arma::mat p = predictors; @@ -100,11 +95,7 @@ void LinearRegression::Predict(const arma::mat& points, { // We want to be sure we have the correct number of dimensions in the // dataset. - // Prevent underflow. - const size_t labels = (parameters.n_rows == 0) ? size_t(0) : - size_t(parameters.n_rows - 1); - util::CheckSameDimensionality(points, labels, "LinearRegression::Predict()", - "points"); + Log::Assert(points.n_rows == parameters.n_rows - 1); // Get the predictions, but this ignores the intercept value // (parameters[0]). predictions = arma::trans(parameters.subvec(1, parameters.n_elem - 1)) @@ -116,8 +107,7 @@ void LinearRegression::Predict(const arma::mat& points, { // We want to be sure we have the correct number of dimensions in // the dataset. - util::CheckSameDimensionality(points, parameters, - "LinearRegression::Predict()", "points"); + Log::Assert(points.n_rows == parameters.n_rows); predictions = arma::trans(parameters) * points; } } @@ -125,9 +115,6 @@ void LinearRegression::Predict(const arma::mat& points, double LinearRegression::ComputeError(const arma::mat& predictors, const arma::rowvec& responses) const { - // Sanity check on data. - util::CheckSameSizes(predictors, responses, "LinearRegression::Train()"); - // Get the number of columns and rows of the dataset. const size_t nCols = predictors.n_cols; const size_t nRows = predictors.n_rows; diff --git a/src/mlpack/methods/perceptron/perceptron.hpp b/src/mlpack/methods/perceptron/perceptron.hpp index 3fdfce6e14..4994588c40 100644 --- a/src/mlpack/methods/perceptron/perceptron.hpp +++ b/src/mlpack/methods/perceptron/perceptron.hpp @@ -68,29 +68,6 @@ class Perceptron const size_t numClasses, const size_t maxIterations = 1000); - /** - * Constructor: construct the perceptron by building the weights matrix, which - * is later used in classification. The number of classes should be specified - * separately, and the labels vector should contain values in the range [0, - * numClasses - 1]. The data::NormalizeLabels() function can be used if the - * labels vector does not contain values in the required range. - * - * This constructor supports weights for each data point. - * - * @param data Input, training data. - * @param labels Labels of dataset. - * @param numClasses Number of classes in the dataset. - * @param instanceWeights Weight vector to use for each training point while - * training. - * @param maxIterations Maximum number of iterations for the perceptron - * learning algorithm. - */ - Perceptron(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const arma::rowvec& instanceWeights, - const size_t maxIterations = 1000); - /** * Alternate constructor which copies parameters from an already initiated * perceptron. diff --git a/src/mlpack/methods/perceptron/perceptron_impl.hpp b/src/mlpack/methods/perceptron/perceptron_impl.hpp index 7373a3843b..46d1ff804a 100644 --- a/src/mlpack/methods/perceptron/perceptron_impl.hpp +++ b/src/mlpack/methods/perceptron/perceptron_impl.hpp @@ -62,32 +62,6 @@ Perceptron::Perceptron( Train(data, labels, numClasses); } -/** - * Constructor: construct the perceptron by building the weights matrix, which - * is later used in classification. The number of classes should be specified - * separately, and the labels vector should contain values in the range [0, - * numClasses - 1]. The data::NormalizeLabels() function can be used if the - * labels vector does not contain values in the required range. - * - * This constructor supports weights for each data point. - */ -template< - typename LearnPolicy, - typename WeightInitializationPolicy, - typename MatType -> -Perceptron::Perceptron( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const arma::rowvec& instanceWeights, - const size_t maxIterations) : - maxIterations(maxIterations) -{ - // Start training. - Train(data, labels, numClasses, instanceWeights); -} - /** * Alternate constructor which copies parameters from an already initiated * perceptron. @@ -134,14 +108,13 @@ void Perceptron::Classify( { arma::vec tempLabelMat; arma::uword maxIndex = 0; - predictedLabels.set_size(test.n_cols); // Could probably be faster if done in batch. for (size_t i = 0; i < test.n_cols; ++i) { tempLabelMat = weights.t() * test.col(i) + biases; tempLabelMat.max(maxIndex); - predictedLabels(i) = maxIndex; + predictedLabels(0, i) = maxIndex; } } diff --git a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp index 7ff52cde88..de28d05e74 100644 --- a/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_learning_impl.hpp @@ -53,9 +53,9 @@ QLearning< // Set up q-learning network. if (learningNetwork.Parameters().is_empty()) - learningNetwork.Reset(); + learningNetwork.ResetParameters(); - targetNetwork.Reset(); + targetNetwork.ResetParameters(); #if ENS_VERSION_MAJOR == 1 this->updater.Initialize(learningNetwork.Parameters().n_rows, diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/dueling_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/dueling_dqn.hpp index a55e6c34ba..c4b26a353f 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/dueling_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/dueling_dqn.hpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/simple_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/simple_dqn.hpp index d4ceb5c22c..54ad55f01e 100644 --- a/src/mlpack/methods/reinforcement_learning/q_networks/simple_dqn.hpp +++ b/src/mlpack/methods/reinforcement_learning/q_networks/simple_dqn.hpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include namespace mlpack { @@ -43,6 +43,7 @@ class SimpleDQN /** * Construct an instance of SimpleDQN class. * + * @param inputDim Number of inputs. * @param h1 Number of neurons in hiddenlayer-1. * @param h2 Number of neurons in hiddenlayer-2. * @param outputDim Number of neurons in output layer. @@ -50,7 +51,8 @@ class SimpleDQN * @param init Specifies the initialization rule for the network. * @param outputLayer Specifies the output layer type for network. */ - SimpleDQN(const int h1, + SimpleDQN(const int inputDim, + const int h1, const int h2, const int outputDim, const bool isNoisy = false, @@ -59,21 +61,21 @@ class SimpleDQN network(outputLayer, init), isNoisy(isNoisy) { - network.Add(new ann::Linear(h1)); - network.Add(new ann::ReLU()); + network.Add(new ann::Linear<>(inputDim, h1)); + network.Add(new ann::ReLULayer<>()); if (isNoisy) { - noisyLayerIndex.push_back(network.Network().size()); - network.Add(new ann::NoisyLinear(h2)); - network.Add(new ann::ReLU()); - noisyLayerIndex.push_back(network.Network().size()); - network.Add(new ann::NoisyLinear(outputDim)); + noisyLayerIndex.push_back(network.Model().size()); + network.Add(new ann::NoisyLinear<>(h1, h2)); + network.Add(new ann::ReLULayer<>()); + noisyLayerIndex.push_back(network.Model().size()); + network.Add(new ann::NoisyLinear<>(h2, outputDim)); } else { - network.Add(new ann::Linear(h2)); - network.Add(new ann::ReLU()); - network.Add(new ann::Linear(outputDim)); + network.Add(new ann::Linear<>(h1, h2)); + network.Add(new ann::ReLULayer<>()); + network.Add(new ann::Linear<>(h2, outputDim)); } } @@ -118,9 +120,9 @@ class SimpleDQN /** * Resets the parameters of the network. */ - void Reset() + void ResetParameters() { - network.Reset(); + network.ResetParameters(); } /** @@ -130,8 +132,8 @@ class SimpleDQN { for (size_t i = 0; i < noisyLayerIndex.size(); i++) { - dynamic_cast( - network.Network()[noisyLayerIndex[i]])->ResetNoise(); + boost::get*> + (network.Model()[noisyLayerIndex[i]])->ResetNoise(); } } diff --git a/src/mlpack/methods/reinforcement_learning/sac.hpp b/src/mlpack/methods/reinforcement_learning/sac.hpp index 55ce00aa9a..27ddb5cba0 100644 --- a/src/mlpack/methods/reinforcement_learning/sac.hpp +++ b/src/mlpack/methods/reinforcement_learning/sac.hpp @@ -19,6 +19,7 @@ #include "replay/random_replay.hpp" #include #include +#include #include "training_config.hpp" namespace mlpack { diff --git a/src/mlpack/prereqs.hpp b/src/mlpack/prereqs.hpp index 6310364f55..cc865820e1 100644 --- a/src/mlpack/prereqs.hpp +++ b/src/mlpack/prereqs.hpp @@ -33,7 +33,6 @@ #include #include #include -#include #include #include @@ -108,6 +107,7 @@ using enable_if_t = typename enable_if::type; #include #include #include +#include #include #include #include @@ -118,10 +118,20 @@ using enable_if_t = typename enable_if::type; #include #include #include +#include +#include #include #include #include +// If we have Boost 1.58 or older and are using C++14, the compilation is likely +// to fail due to boost::visitor issues. We will pre-emptively fail. +#if __cplusplus > 201103L && BOOST_VERSION < 105900 +#error Use of C++14 mode with Boost < 1.59 is known to cause compilation \ +problems. Instead specify the C++11 standard (-std=c++11 with gcc or clang), \ +or upgrade Boost to 1.59 or newer. +#endif + // On Visual Studio, disable C4519 (default arguments for function templates) // since it's by default an error, which doesn't even make any sense because // it's part of the C++11 standard. diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 6e6b0840b5..b27b15619d 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -11,9 +11,10 @@ add_executable(mlpack_test ann_layer_test.cpp ann_regularizer_test.cpp ann_test_tools.hpp + ann_visitor_test.cpp armadillo_svd_test.cpp arma_extend_test.cpp -# async_learning_test.cpp + async_learning_test.cpp augmented_rnns_tasks_test.cpp bayesian_linear_regression_test.cpp bias_svd_test.cpp @@ -27,7 +28,7 @@ add_executable(mlpack_test cosine_tree_test.cpp cv_test.cpp dbscan_test.cpp -# dcgan_test.cpp + dcgan_test.cpp decision_tree_regressor_test.cpp decision_tree_test.cpp det_test.cpp @@ -39,7 +40,7 @@ add_executable(mlpack_test fastmks_test.cpp feedforward_network_test.cpp feedforward_network_2_test.cpp -# gan_test.cpp + gan_test.cpp gmm_test.cpp hmm_test.cpp hpt_test.cpp @@ -59,6 +60,7 @@ add_executable(mlpack_test krann_search_test.cpp ksinit_test.cpp lars_test.cpp + layer_names_test.cpp lin_alg_test.cpp linear_regression_test.cpp lmnn_test.cpp @@ -88,18 +90,18 @@ add_executable(mlpack_test python_binding_test.cpp qdafn_test.cpp quic_svd_test.cpp -# q_learning_test.cpp + q_learning_test.cpp radical_test.cpp random_forest_test.cpp random_test.cpp randomized_svd_test.cpp range_search_test.cpp -# rbm_network_test.cpp + rbm_network_test.cpp rectangle_tree_test.cpp recurrent_network_test.cpp -# rnn_reber_test.cpp + rnn_reber_test.cpp regularized_svd_test.cpp -# reward_clipping_test.cpp + reward_clipping_test.cpp rl_components_test.cpp scaling_test.cpp size_checks_test.cpp @@ -128,7 +130,7 @@ add_executable(mlpack_test ub_tree_test.cpp union_find_test.cpp vantage_point_tree_test.cpp -# wgan_test.cpp + wgan_test.cpp xgboost_test.cpp main_tests/adaboost_test.cpp main_tests/adaboost_train_test.cpp diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 2cadc4375c..1c4458c4f9 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -12,7 +12,7 @@ */ #include -#include +#include #include #include #include @@ -135,7 +135,7 @@ void CheckInverseCorrect(const arma::colvec input) * * @param input Input data used for evaluating the HardTanH activation function. * @param target Target data used to evaluate the HardTanH activation. - * + */ void CheckHardTanHActivationCorrect(const arma::colvec input, const arma::colvec target) { @@ -148,7 +148,7 @@ void CheckHardTanHActivationCorrect(const arma::colvec input, { REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the HardTanH activation function derivative test. The @@ -157,7 +157,7 @@ void CheckHardTanHActivationCorrect(const arma::colvec input, * @param input Input data used for evaluating the HardTanH activation * function. * @param target Target data used to evaluate the HardTanH activation. - * + */ void CheckHardTanHDerivativeCorrect(const arma::colvec input, const arma::colvec target) { @@ -174,7 +174,7 @@ void CheckHardTanHDerivativeCorrect(const arma::colvec input, { REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the LeakyReLU activation function test. The function is @@ -187,7 +187,7 @@ void CheckHardTanHDerivativeCorrect(const arma::colvec input, void CheckLeakyReLUActivationCorrect(const arma::colvec input, const arma::colvec target) { - LeakyReLU lrf; + LeakyReLU<> lrf; // Test the activation function using the entire vector as input. arma::colvec activations; @@ -210,7 +210,7 @@ void CheckLeakyReLUActivationCorrect(const arma::colvec input, void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, const arma::colvec target) { - LeakyReLU lrf; + LeakyReLU<> lrf; // Test the calculation of the derivatives using the entire vector as input. arma::colvec derivatives; @@ -230,7 +230,7 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, * * @param input Input data used for evaluating the ELU activation function. * @param target Target data used to evaluate the ELU activation. - * + */ void CheckELUActivationCorrect(const arma::colvec input, const arma::colvec target) { @@ -244,7 +244,7 @@ void CheckELUActivationCorrect(const arma::colvec input, { REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the ELU activation function derivative test. The function @@ -252,7 +252,7 @@ void CheckELUActivationCorrect(const arma::colvec input, * * @param input Input data used for evaluating the ELU activation function. * @param target Target data used to evaluate the ELU activation. - * + */ void CheckELUDerivativeCorrect(const arma::colvec input, const arma::colvec target) { @@ -270,7 +270,7 @@ void CheckELUDerivativeCorrect(const arma::colvec input, { REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the PReLU activation function test. The function @@ -279,7 +279,7 @@ void CheckELUDerivativeCorrect(const arma::colvec input, * @param input Input data used for evaluating the PReLU activation * function. * @param target Target data used to evaluate the PReLU activation. - * + */ void CheckPReLUActivationCorrect(const arma::colvec input, const arma::colvec target) { @@ -292,7 +292,7 @@ void CheckPReLUActivationCorrect(const arma::colvec input, { REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the PReLU activation function derivative test. @@ -302,7 +302,7 @@ void CheckPReLUActivationCorrect(const arma::colvec input, * @param input Input data used for evaluating the PReLU activation * function. * @param target Target data used to evaluate the PReLU activation. - * + */ void CheckPReLUDerivativeCorrect(const arma::colvec input, const arma::colvec target) { @@ -318,7 +318,7 @@ void CheckPReLUDerivativeCorrect(const arma::colvec input, { REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the PReLU activation function gradient test. @@ -328,7 +328,7 @@ void CheckPReLUDerivativeCorrect(const arma::colvec input, * @param input Input data used for evaluating the PReLU activation * function. * @param target Target data used to evaluate the PReLU gradient. - * + */ void CheckPReLUGradientCorrect(const arma::colvec input, const arma::colvec target) { @@ -343,7 +343,7 @@ void CheckPReLUGradientCorrect(const arma::colvec input, REQUIRE(gradient.n_rows == 1); REQUIRE(gradient.n_cols == 1); REQUIRE(gradient(0) == Approx(target(0)).epsilon(1e-5)); -}*/ +} /** * Implementation of the Hard Shrink activation function test. The function is @@ -351,7 +351,7 @@ void CheckPReLUGradientCorrect(const arma::colvec input, * * @param input Input data used for evaluating the Hard Shrink activation function. * @param target Target data used to evaluate the Hard Shrink activation. - * + */ void CheckHardShrinkActivationCorrect(const arma::colvec input, const arma::colvec target) { @@ -364,7 +364,7 @@ void CheckHardShrinkActivationCorrect(const arma::colvec input, { REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the HardShrink activation function derivative test. @@ -374,7 +374,7 @@ void CheckHardShrinkActivationCorrect(const arma::colvec input, * @param input Input data used for evaluating the HardShrink activation * function. * @param target Target data used to evaluate the HardShrink activation. - * + */ void CheckHardShrinkDerivativeCorrect(const arma::colvec input, const arma::colvec target) { @@ -390,7 +390,7 @@ void CheckHardShrinkDerivativeCorrect(const arma::colvec input, { REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the Soft Shrink activation function test. The function is @@ -399,7 +399,7 @@ void CheckHardShrinkDerivativeCorrect(const arma::colvec input, * @param input Input data used for evaluating the Soft Shrink activation * function. * @param target Target data used to evaluate the Soft Shrink activation. - * + */ void CheckSoftShrinkActivationCorrect(const arma::colvec input, const arma::colvec target) { @@ -412,7 +412,7 @@ void CheckSoftShrinkActivationCorrect(const arma::colvec input, { REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the Soft Shrink activation function derivative test. @@ -422,7 +422,7 @@ void CheckSoftShrinkActivationCorrect(const arma::colvec input, * @param input Input data used for evaluating the Soft Shrink activation * function. * @param target Target data used to evaluate the Soft Shrink activation. - * + */ void CheckSoftShrinkDerivativeCorrect(const arma::colvec input, const arma::colvec target) { @@ -438,12 +438,12 @@ void CheckSoftShrinkDerivativeCorrect(const arma::colvec input, { REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Simple SELU activation test to check whether the mean and variance remain * invariant after passing normalized inputs through the function. - * + */ TEST_CASE("SELUFunctionNormalizedTest", "[ActivationFunctionsTest]") { arma::mat input = arma::randn(1000, 1); @@ -459,12 +459,12 @@ TEST_CASE("SELUFunctionNormalizedTest", "[ActivationFunctionsTest]") REQUIRE(arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))) <= 0.1); -}*/ +} /** * Simple SELU activation test to check whether the mean and variance * vary significantly after passing unnormalized inputs through the function. - * + */ TEST_CASE("SELUFunctionUnnormalizedTest", "[ActivationFunctionsTest]") { const arma::colvec input("5.96402758 0.9966824 0.99975321 1 \ @@ -481,13 +481,13 @@ TEST_CASE("SELUFunctionUnnormalizedTest", "[ActivationFunctionsTest]") REQUIRE(arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))) >= 0.1); -}*/ +} /** * Simple SELU derivative test to check whether the derivatives * produced by the activation function are correct. * - * + */ TEST_CASE("SELUFunctionDerivativeTest", "[ActivationFunctionsTest]") { arma::mat input = arma::ones(1000, 1); @@ -511,7 +511,7 @@ TEST_CASE("SELUFunctionDerivativeTest", "[ActivationFunctionsTest]") REQUIRE(arma::as_scalar(arma::abs(arma::mean(derivatives) - selu.Lambda() * selu.Alpha() - arma::mean(activations))) <= 10e-4); -}*/ +} /** * Implementation of the CELU activation function test. The function is @@ -519,7 +519,7 @@ TEST_CASE("SELUFunctionDerivativeTest", "[ActivationFunctionsTest]") * * @param input Input data used for evaluating the CELU activation function. * @param target Target data used to evaluate the CELU activation. - * + */ void CheckCELUActivationCorrect(const arma::colvec input, const arma::colvec target) { @@ -533,7 +533,7 @@ void CheckCELUActivationCorrect(const arma::colvec input, { REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the CELU activation function derivative test. The function @@ -541,7 +541,7 @@ void CheckCELUActivationCorrect(const arma::colvec input, * * @param input Input data used for evaluating the CELU activation function. * @param target Target data used to evaluate the CELU activation. - * + */ void CheckCELUDerivativeCorrect(const arma::colvec input, const arma::colvec target) { @@ -559,7 +559,7 @@ void CheckCELUDerivativeCorrect(const arma::colvec input, { REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the ISRLU activation function test. The function is @@ -567,7 +567,7 @@ void CheckCELUDerivativeCorrect(const arma::colvec input, * * @param input Input data used for evaluating the ISRLU activation function. * @param target Target data used to evaluate the ISRLU activation. - * + */ void CheckISRLUActivationCorrect(const arma::colvec input, const arma::colvec target) { @@ -581,7 +581,7 @@ void CheckISRLUActivationCorrect(const arma::colvec input, { REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the ISRLU activation function derivative test. The function @@ -589,7 +589,7 @@ void CheckISRLUActivationCorrect(const arma::colvec input, * * @param input Input data used for evaluating the ISRLU activation function. * @param target Target data used to evaluate the ISRLU activation. - * + */ void CheckISRLUDerivativeCorrect(const arma::colvec input, const arma::colvec target) { @@ -607,7 +607,7 @@ void CheckISRLUDerivativeCorrect(const arma::colvec input, { REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the Softmin activation function test. The function is @@ -615,7 +615,7 @@ void CheckISRLUDerivativeCorrect(const arma::colvec input, * * @param input Input data used for evaluating the Softmin activation function. * @param target Target data used to evaluate the Softmin activation. - * + */ void CheckSoftminActivationCorrect(const arma::colvec input, const arma::colvec target) { @@ -629,7 +629,7 @@ void CheckSoftminActivationCorrect(const arma::colvec input, { REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the Softmin activation function derivative test. @@ -637,7 +637,7 @@ void CheckSoftminActivationCorrect(const arma::colvec input, * * @param input Input data used for evaluating the Softmin activation function. * @param target Target data used to evaluate the Softmin activation. - * + */ void CheckSoftminDerivativeCorrect(const arma::colvec input, const arma::colvec target) { @@ -658,7 +658,7 @@ void CheckSoftminDerivativeCorrect(const arma::colvec input, { REQUIRE(derivatives.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the Flatten T Swish activation function test. The function is @@ -667,7 +667,7 @@ void CheckSoftminDerivativeCorrect(const arma::colvec input, * @param input Input data used for evaluating the Flatten T Swish activation * function. * @param target Target data used to evaluate the Flatten T Swish activation. - * + */ void CheckFlattenTSwishActivationCorrect(const arma::colvec input, const arma::colvec target) { @@ -679,7 +679,7 @@ void CheckFlattenTSwishActivationCorrect(const arma::colvec input, { REQUIRE(activations.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the Softmin activation function derivative test. @@ -687,7 +687,7 @@ void CheckFlattenTSwishActivationCorrect(const arma::colvec input, * * @param input Input data used for evaluating the Softmin activation function. * @param target Target data used to evaluate the Softmin activation. - * + */ void CheckFlattenTSwishDerivateCorrect(const arma::colvec input, const arma::colvec target) { @@ -702,7 +702,7 @@ void CheckFlattenTSwishDerivateCorrect(const arma::colvec input, { REQUIRE(derivate.at(i) == Approx(target.at(i)).epsilon(1e-5)); } -}*/ +} /** * Implementation of the ReLU6 activation function derivative test. The function @@ -710,7 +710,7 @@ void CheckFlattenTSwishDerivateCorrect(const arma::colvec input, * * @param input Input data used for evaluating the ReLU6 activation function. * @param target Target data used to evaluate the ReLU6 activation. - * + */ void CheckReLU6Correct(const arma::colvec input, const arma::colvec ActivationTarget, const arma::colvec DerivativeTarget) @@ -733,11 +733,11 @@ void CheckReLU6Correct(const arma::colvec input, { REQUIRE(derivatives.at(i) == Approx(DerivativeTarget.at(i)).epsilon(1e-5)); } -}*/ +} /** * Basic test of the ReLU6 function. - * + */ TEST_CASE("ReLU6FunctionTest", "[ActivationFunctionsTest]") { const arma::colvec activationData("-2.0 3.0 0.0 6.0 24.0"); @@ -749,7 +749,7 @@ TEST_CASE("ReLU6FunctionTest", "[ActivationFunctionsTest]") const arma::colvec desiredDerivatives("0.0 1.0 0.0 0.0 0.0"); CheckReLU6Correct(activationData, desiredActivations, desiredDerivatives); -}*/ +} /** * Basic test of the tanh function. @@ -847,7 +847,7 @@ TEST_CASE("LeakyReLUFunctionTest", "[ActivationFunctionsTest]") /** * Basic test of the HardTanH function. - * + */ TEST_CASE("HardTanHFunctionTest", "[ActivationFunctionsTest]") { const arma::colvec desiredActivations("-1 1 1 -1 \ @@ -858,11 +858,11 @@ TEST_CASE("HardTanHFunctionTest", "[ActivationFunctionsTest]") CheckHardTanHActivationCorrect(activationData, desiredActivations); CheckHardTanHDerivativeCorrect(activationData, desiredDerivatives); -}*/ +} /** * Basic test of the ELU function. - * + */ TEST_CASE("ELUFunctionTest", "[ActivationFunctionsTest]") { const arma::colvec desiredActivations("-0.86466471 3.2 4.5 -1.0 \ @@ -873,7 +873,7 @@ TEST_CASE("ELUFunctionTest", "[ActivationFunctionsTest]") CheckELUActivationCorrect(activationData, desiredActivations); CheckELUDerivativeCorrect(activationData, desiredDerivatives); -}*/ +} /** * Basic test of the softplus function. @@ -898,7 +898,7 @@ TEST_CASE("SoftplusFunctionTest", "[ActivationFunctionsTest]") /** * Basic test of the PReLU function. - * + */ TEST_CASE("PReLUFunctionTest", "[ActivationFunctionsTest]") { const arma::colvec desiredActivations("-0.06 3.2 4.5 -3.006 \ @@ -911,11 +911,11 @@ TEST_CASE("PReLUFunctionTest", "[ActivationFunctionsTest]") CheckPReLUActivationCorrect(activationData, desiredActivations); CheckPReLUDerivativeCorrect(desiredActivations, desiredDerivatives); CheckPReLUGradientCorrect(activationData, desiredGradient); -}*/ +} /** * Basic test of the CReLU function. - * + */ TEST_CASE("CReLUFunctionTest", "[ActivationFunctionsTest]") { const arma::colvec desiredActivations("0 3.2 4.5 0 \ @@ -942,11 +942,11 @@ TEST_CASE("CReLUFunctionTest", "[ActivationFunctionsTest]") REQUIRE(derivatives.at(i) == Approx(desiredDerivatives.at(i)).epsilon(1e-5)); } -}*/ +} /** * Basic test of the swish function. - * + */ TEST_CASE("SwishFunctionTest", "[ActivationFunctionsTest]") { // Hand-calculated values using Python interpreter. @@ -961,7 +961,7 @@ TEST_CASE("SwishFunctionTest", "[ActivationFunctionsTest]") CheckActivationCorrect(activationData, desiredActivations); CheckDerivativeCorrect(desiredActivations, desiredDerivatives); -}*/ +} /** * Basic test of the hard sigmoid function. @@ -1049,7 +1049,7 @@ TEST_CASE("GELUFunctionTest", "[ActivationFunctionsTest]") /** * Basic test of the Hard Shrink function. - * + */ TEST_CASE("HardShrinkFunctionTest", "[ActivationFunctionsTest]") { const arma::colvec desiredActivations("-2 3.2 4.5 -100.2 1 -1 2 0"); @@ -1060,7 +1060,7 @@ TEST_CASE("HardShrinkFunctionTest", "[ActivationFunctionsTest]") desiredActivations); CheckHardShrinkDerivativeCorrect(desiredActivations, desiredDerivatives); -}*/ +} /** * Basic test of the Elliot function. @@ -1104,7 +1104,7 @@ TEST_CASE("ElishFunctionTest", "[ActivationFunctionsTest]") /** * Basic test of the Soft Shrink function. - * + */ TEST_CASE("SoftShrinkFunctionTest", "[ActivationFunctionsTest]") { const arma::colvec desiredActivations("-1.5 2.7 4 -99.7 0.5 -0.5 1.5 0"); @@ -1115,11 +1115,11 @@ TEST_CASE("SoftShrinkFunctionTest", "[ActivationFunctionsTest]") desiredActivations); CheckSoftShrinkDerivativeCorrect(desiredActivations, desiredDerivatives); -}*/ +} /** * Basic test of the CELU activation function. - * + */ TEST_CASE("CELUFunctionTest", "[ActivationFunctionsTest]") { const arma::colvec desiredActivations("-0.86466472 3.2 4.5 \ @@ -1131,11 +1131,11 @@ TEST_CASE("CELUFunctionTest", "[ActivationFunctionsTest]") CheckCELUActivationCorrect(activationData, desiredActivations); CheckCELUDerivativeCorrect(desiredActivations, desiredDerivatives); -}*/ +} /** * Basic test of the ISRLU activation function. - * + */ TEST_CASE("ISRLUFunctionTest", "[ActivationFunctionsTest]") { const arma::colvec desiredActivations("-0.89442719 3.2 4.5 \ @@ -1147,7 +1147,7 @@ TEST_CASE("ISRLUFunctionTest", "[ActivationFunctionsTest]") CheckISRLUActivationCorrect(activationData, desiredActivations); CheckISRLUDerivativeCorrect(activationData, desiredDerivatives); -}*/ +} /** * Basic test of the inverse quadratic function. @@ -1275,7 +1275,7 @@ TEST_CASE("GaussianFunctionTest", "[ActivationFunctionsTest]") /** * Basic test of the Softmin function. - * + */ TEST_CASE("SoftminFunctionTest", "[ActivationFunctionsTest]") { const arma::colvec activationData("4.2 2.4 7.0 6.4"); @@ -1291,7 +1291,7 @@ TEST_CASE("SoftminFunctionTest", "[ActivationFunctionsTest]") desiredActivations); CheckSoftminDerivativeCorrect(activationData, desiredDerivatives); -}*/ +} /** * Basic test of the Hard Swish function. @@ -1360,7 +1360,7 @@ TEST_CASE("SILUFunctionTest", "[ActivationFunctionsTest]") /** * Basic test of Flatten T Swish function. - * + */ TEST_CASE("FlattenTSwishFunctionTest", "[ActivationFunctionsTest]") { // Random Value. @@ -1378,4 +1378,4 @@ TEST_CASE("FlattenTSwishFunctionTest", "[ActivationFunctionsTest]") CheckFlattenTSwishActivationCorrect(input, desiredActivation); CheckFlattenTSwishDerivateCorrect(desiredActivation, desiredDerivation); -}*/ +} diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index cb1c5d766f..b9d5452c41 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "test_catch_tools.hpp" #include "catch.hpp" @@ -30,59 +31,59 @@ using namespace mlpack; using namespace mlpack::ann; -// // network1 should be allocated with `new`, and trained on some data. -// template -// void CheckRNNCopyFunction(ModelType* network1, -// MatType& trainData, -// MatType& trainLabels, -// const size_t maxEpochs) -// { -// arma::cube predictions1; -// arma::cube predictions2; -// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); +// network1 should be allocated with `new`, and trained on some data. +template +void CheckRNNCopyFunction(ModelType* network1, + MatType& trainData, + MatType& trainLabels, + const size_t maxEpochs) +{ + arma::cube predictions1; + arma::cube predictions2; + ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); -// network1->Train(trainData, trainLabels, opt); -// network1->Predict(trainData, predictions1); + network1->Train(trainData, trainLabels, opt); + network1->Predict(trainData, predictions1); -// RNN<> network2 = *network1; -// delete network1; + RNN<> network2 = *network1; + delete network1; -// // Deallocating all of network1's memory, so that network2 does not use any -// // of that memory. -// network2.Predict(trainData, predictions2); -// CheckMatrices(predictions1, predictions2); -// } + // Deallocating all of network1's memory, so that network2 does not use any + // of that memory. + network2.Predict(trainData, predictions2); + CheckMatrices(predictions1, predictions2); +} -// // network1 should be allocated with `new`, and trained on some data. -// template -// void CheckRNNMoveFunction(ModelType* network1, -// MatType& trainData, -// MatType& trainLabels, -// const size_t maxEpochs) -// { -// arma::cube predictions1; -// arma::cube predictions2; -// ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); +// network1 should be allocated with `new`, and trained on some data. +template +void CheckRNNMoveFunction(ModelType* network1, + MatType& trainData, + MatType& trainLabels, + const size_t maxEpochs) +{ + arma::cube predictions1; + arma::cube predictions2; + ens::StandardSGD opt(0.1, 1, maxEpochs * trainData.n_slices, -100, false); -// network1->Train(trainData, trainLabels, opt); -// network1->Predict(trainData, predictions1); + network1->Train(trainData, trainLabels, opt); + network1->Predict(trainData, predictions1); -// RNN<> network2(std::move(*network1)); -// delete network1; + RNN<> network2(std::move(*network1)); + delete network1; -// // Deallocating all of network1's memory, so that network2 does not use any -// // of that memory. -// network2.Predict(trainData, predictions2); -// CheckMatrices(predictions1, predictions2); -// } + // Deallocating all of network1's memory, so that network2 does not use any + // of that memory. + network2.Predict(trainData, predictions2); + CheckMatrices(predictions1, predictions2); +} /** * Simple add module test. - * + */ TEST_CASE("SimpleAddLayerTest", "[ANNLayerTest]") { arma::mat output, input, delta; - Add module(10); + Add<> module(10); module.Parameters().randu(); // Test the Forward function. @@ -104,11 +105,10 @@ TEST_CASE("SimpleAddLayerTest", "[ANNLayerTest]") module.Backward(input, output, delta); REQUIRE(arma::accu(output) == Approx(arma::accu(delta)).epsilon(1e-5)); } -*/ /** * Jacobian add module test. - * + */ TEST_CASE("JacobianAddLayerTest", "[ANNLayerTest]") { for (size_t i = 0; i < 5; ++i) @@ -117,18 +117,17 @@ TEST_CASE("JacobianAddLayerTest", "[ANNLayerTest]") arma::mat input; input.set_size(elements, 1); - Add module(elements); + Add<> module(elements); module.Parameters().randu(); double error = JacobianTest(module, input); REQUIRE(error <= 1e-5); } } -*/ /** * Add layer numerical gradient test. - * + */ TEST_CASE("GradientAddLayerTest", "[ANNLayerTest]") { // Add function gradient instantiation. @@ -138,12 +137,13 @@ TEST_CASE("GradientAddLayerTest", "[ANNLayerTest]") input(arma::randu(10, 1)), target(arma::mat("0")) { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 10); - model->Add(10); - model->Add(); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 10); + model->Add >(10); + model->Add >(); } ~GradientFunction() @@ -160,33 +160,33 @@ TEST_CASE("GradientAddLayerTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, NguyenWidrowInitialization>* model; arma::mat input, target; } function; REQUIRE(CheckGradient(function) <= 1e-4); -}*/ +} /** * Test that the function that can access the outSize parameter of * the Add layer works. - * + */ TEST_CASE("AddLayerParametersTest", "[ANNLayerTest]") { // Parameter : outSize. - Add layer(7); + Add<> layer(7); // Make sure we can get the parameter successfully. REQUIRE(layer.OutputSize() == 7); -}*/ +} /** * Simple constant module test. - * + */ TEST_CASE("SimpleConstantLayerTest", "[ANNLayerTest]") { arma::mat output, input, delta; - Constant module(10, 3.0); + Constant<> module(10, 3.0); // Test the Forward function. input = arma::zeros(10, 1); @@ -205,11 +205,11 @@ TEST_CASE("SimpleConstantLayerTest", "[ANNLayerTest]") // Test the backward function. module.Backward(input, output, delta); REQUIRE(arma::accu(delta) == 0); -}*/ +} /** * Jacobian constant module test. - * + */ TEST_CASE("JacobianConstantLayerTest", "[ANNLayerTest]") { for (size_t i = 0; i < 5; ++i) @@ -218,25 +218,25 @@ TEST_CASE("JacobianConstantLayerTest", "[ANNLayerTest]") arma::mat input; input.set_size(elements, 1); - Constant module(elements, 1.0); + Constant<> module(elements, 1.0); double error = JacobianTest(module, input); REQUIRE(error <= 1e-5); } -}*/ +} /** * Test that the function that can access the outSize parameter of the * Constant layer works. - * + */ TEST_CASE("ConstantLayerParametersTest", "[ANNLayerTest]") { // Parameter : outSize. - Constant layer(7); + Constant<> layer(7); // Make sure we can get the parameter successfully. REQUIRE(layer.OutSize() == 7); -}*/ +} /** * Simple dropout module test. @@ -250,8 +250,8 @@ TEST_CASE("SimpleDropoutLayerTest", "[ANNLayerTest]") arma::mat input(1000, 1); input.fill(1 - p); - Dropout module(p); - module.Training() = true; + Dropout<> module(p); + module.Deterministic() = false; // Test the Forward function. arma::mat output; @@ -264,7 +264,7 @@ TEST_CASE("SimpleDropoutLayerTest", "[ANNLayerTest]") REQUIRE(arma::as_scalar(arma::abs(arma::mean(delta) - (1 - p))) <= 0.05); // Test the Forward function. - module.Training() = false; + module.Deterministic() = true; module.Forward(input, output); REQUIRE(arma::accu(input) == arma::accu(output)); } @@ -285,8 +285,8 @@ TEST_CASE("DropoutProbabilityTest", "[ANNLayerTest]") double nonzeroCount = 0; for (size_t i = 0; i < iterations; ++i) { - Dropout module(probability[trial]); - module.Training() = true; + Dropout<> module(probability[trial]); + module.Deterministic() = false; arma::mat output; module.Forward(input, output); @@ -310,8 +310,8 @@ TEST_CASE("DropoutProbabilityTest", "[ANNLayerTest]") TEST_CASE("NoDropoutTest", "[ANNLayerTest]") { arma::mat input = arma::ones(1500, 1); - Dropout module(0); - module.Training() = true; + Dropout<> module(0); + module.Deterministic() = false; arma::mat output; module.Forward(input, output); @@ -332,11 +332,11 @@ TEST_CASE("SimpleAlphaDropoutLayerTest", "[ANNLayerTest]") // and variance nearabout 1. arma::mat input = arma::randn(1000, 1); - AlphaDropout module(p); - module.Training() = true; + AlphaDropout<> module(p); + module.Deterministic() = false; // Test the Forward function when training phase. - arma::mat output(arma::size(input)); + arma::mat output; module.Forward(input, output); // Check whether mean remains nearly same. REQUIRE(arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))) <= @@ -352,7 +352,7 @@ TEST_CASE("SimpleAlphaDropoutLayerTest", "[ANNLayerTest]") REQUIRE(arma::as_scalar(arma::abs(arma::mean(delta) - 0)) <= 0.05); // Test the Forward function when testing phase. - module.Training() = false; + module.Deterministic() = true; module.Forward(input, output); REQUIRE(arma::accu(input) == arma::accu(output)); } @@ -373,10 +373,10 @@ TEST_CASE("AlphaDropoutProbabilityTest", "[ANNLayerTest]") double nonzeroCount = 0; for (size_t i = 0; i < iterations; ++i) { - AlphaDropout module(probability[trial]); - module.Training() = true; + AlphaDropout<> module(probability[trial]); + module.Deterministic() = false; - arma::mat output(arma::size(input)); + arma::mat output; module.Forward(input, output); // Return a column vector containing the indices of elements of X @@ -401,8 +401,8 @@ TEST_CASE("AlphaDropoutProbabilityTest", "[ANNLayerTest]") TEST_CASE("NoAlphaDropoutTest", "[ANNLayerTest]") { arma::mat input = arma::ones(1500, 1); - AlphaDropout module(0); - module.Training() = false; + AlphaDropout<> module(0); + module.Deterministic() = false; arma::mat output; module.Forward(input, output); @@ -410,89 +410,90 @@ TEST_CASE("NoAlphaDropoutTest", "[ANNLayerTest]") REQUIRE(arma::accu(output) == arma::accu(input)); } -// /** -// * Simple linear module test. -// */ -// TEST_CASE("SimpleLinearLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// Linear<> module(10, 10); -// module.Parameters().randu(); -// module.Reset(); +/** + * Simple linear module test. + */ +TEST_CASE("SimpleLinearLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + Linear<> module(10, 10); + module.Parameters().randu(); + module.Reset(); -// // Test the Forward function. -// input = arma::zeros(10, 1); -// module.Forward(input, output); -// REQUIRE(arma::accu(module.Parameters().submat(100, -// 0, module.Parameters().n_elem - 1, 0)) == -// Approx(arma::accu(output)).epsilon(1e-5)); + // Test the Forward function. + input = arma::zeros(10, 1); + module.Forward(input, output); + REQUIRE(arma::accu(module.Parameters().submat(100, + 0, module.Parameters().n_elem - 1, 0)) == + Approx(arma::accu(output)).epsilon(1e-5)); -// // Test the Backward function. -// module.Backward(input, input, delta); -// REQUIRE(arma::accu(delta) == 0); -// } + // Test the Backward function. + module.Backward(input, input, delta); + REQUIRE(arma::accu(delta) == 0); +} -// /** -// * Jacobian linear module test. -// */ -// TEST_CASE("JacobianLinearLayerTest", "[ANNLayerTest]") -// { -// for (size_t i = 0; i < 5; ++i) -// { -// const size_t inputElements = math::RandInt(2, 1000); -// const size_t outputElements = math::RandInt(2, 1000); +/** + * Jacobian linear module test. + */ +TEST_CASE("JacobianLinearLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + const size_t inputElements = math::RandInt(2, 1000); + const size_t outputElements = math::RandInt(2, 1000); -// arma::mat input; -// input.set_size(inputElements, 1); + arma::mat input; + input.set_size(inputElements, 1); -// Linear<> module(inputElements, outputElements); -// module.Parameters().randu(); + Linear<> module(inputElements, outputElements); + module.Parameters().randu(); -// double error = JacobianTest(module, input); -// REQUIRE(error <= 1e-5); -// } -// } + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); + } +} -// /** -// * Linear layer numerical gradient test. -// */ -// TEST_CASE("GradientLinearLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("1")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(10, 10); -// model->Add >(10, 2); -// model->Add >(); -// } +/** + * Linear layer numerical gradient test. + */ +TEST_CASE("GradientLinearLayerTest", "[ANNLayerTest]") +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 10); + model->Add >(10, 2); + model->Add >(); + } -// ~GradientFunction() -// { -// delete model; -// } + ~GradientFunction() + { + delete model; + } -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } -// arma::mat& Parameters() { return model->Parameters(); } + arma::mat& Parameters() { return model->Parameters(); } -// FFN* model; -// arma::mat input, target; -// } function; + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; -// REQUIRE(CheckGradient(function) <= 1e-4); -// } + REQUIRE(CheckGradient(function) <= 1e-4); +} /** * Simple Linear3D layer test. @@ -505,26 +506,18 @@ TEST_CASE("SimpleLinear3DLayerTest", "[ANNLayerTest]") const size_t batchSize = 1; arma::mat input, output, delta; - // Create a Linear3D layer outside of a network, and then set its memory. - Linear3D module(outSize); - module.InputDimensions() = std::vector({ 4, 2 }); - module.ComputeOutputDimensions(); - arma::mat weights(module.WeightSize(), 1); - module.SetWeights(weights.memptr()); - + Linear3D<> module(inSize, outSize); + module.Reset(); module.Parameters().randu(); // Test the Forward function. input = arma::zeros(inSize * nPoints, batchSize); - output.set_size(outSize * nPoints, batchSize); module.Forward(input, output); REQUIRE(arma::accu(module.Bias()) == Approx(arma::accu(output) / (nPoints * batchSize)).epsilon(1e-3)); // Test the Backward function. - delta.set_size(input.n_rows, input.n_cols); - output.zeros(); - module.Backward(input, output, delta); + module.Backward(input, input, delta); REQUIRE(arma::accu(delta) == 0); } @@ -543,13 +536,7 @@ TEST_CASE("JacobianLinear3DLayerTest", "[ANNLayerTest]") arma::mat input; input.set_size(inSize * nPoints, batchSize); - // Create a Linear3D layer outside a network and initialize its memory. - Linear3D module(outSize); - module.InputDimensions() = std::vector({ inSize, nPoints }); - module.ComputeOutputDimensions(); - arma::mat weights(module.WeightSize(), 1); - module.SetWeights(weights.memptr()); - + Linear3D<> module(inSize, outSize); module.Parameters().randu(); double error = JacobianTest(module, input); @@ -578,10 +565,11 @@ TEST_CASE("GradientLinear3DLayerTest", "[ANNLayerTest]") target(1, 1) = 1; target(1, 2) = 1; - model = new FFN(); - model->ResetData(input, target); - model->Add(outSize); - model->InputDimensions() = std::vector{ 4, 2 }; + model = new FFN, RandomInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add>(); + model->Add>(inSize, outSize); } ~GradientFunction() @@ -598,7 +586,7 @@ TEST_CASE("GradientLinear3DLayerTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, RandomInitialization>* model; arma::mat input, target; const size_t inSize; const size_t outSize; @@ -609,79 +597,80 @@ TEST_CASE("GradientLinear3DLayerTest", "[ANNLayerTest]") REQUIRE(CheckGradient(function) <= 1e-7); } -// /** -// * Simple noisy linear module test. -// */ -// TEST_CASE("SimpleNoisyLinearLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// NoisyLinear<> module(10, 10); -// module.Parameters().randu(); -// module.Reset(); +/** + * Simple noisy linear module test. + */ +TEST_CASE("SimpleNoisyLinearLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + NoisyLinear<> module(10, 10); + module.Parameters().randu(); + module.Reset(); -// // Test the Backward function. -// module.Backward(input, input, delta); -// REQUIRE(arma::accu(delta) == 0); -// } + // Test the Backward function. + module.Backward(input, input, delta); + REQUIRE(arma::accu(delta) == 0); +} -// /** -// * Jacobian noisy linear module test. -// */ -// TEST_CASE("JacobianNoisyLinearLayerTest", "[ANNLayerTest]") -// { -// const size_t inputElements = math::RandInt(2, 1000); -// const size_t outputElements = math::RandInt(2, 1000); +/** + * Jacobian noisy linear module test. + */ +TEST_CASE("JacobianNoisyLinearLayerTest", "[ANNLayerTest]") +{ + const size_t inputElements = math::RandInt(2, 1000); + const size_t outputElements = math::RandInt(2, 1000); -// arma::mat input; -// input.set_size(inputElements, 1); + arma::mat input; + input.set_size(inputElements, 1); -// NoisyLinear<> module(inputElements, outputElements); -// module.Parameters().randu(); + NoisyLinear<> module(inputElements, outputElements); + module.Parameters().randu(); -// double error = JacobianTest(module, input); -// REQUIRE(error <= 1e-5); -// } + double error = JacobianTest(module, input); + REQUIRE(error <= 1e-5); +} -// /** -// * Noisy Linear layer numerical gradient test. -// */ -// TEST_CASE("GradientNoisyLinearLayerTest", "[ANNLayerTest]") -// { -// // Noisy linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("1")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(10, 10); -// model->Add >(10, 2); -// model->Add >(); -// } +/** + * Noisy Linear layer numerical gradient test. + */ +TEST_CASE("GradientNoisyLinearLayerTest", "[ANNLayerTest]") +{ + // Noisy linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 10); + model->Add >(10, 2); + model->Add >(); + } -// ~GradientFunction() -// { -// delete model; -// } + ~GradientFunction() + { + delete model; + } -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } -// arma::mat& Parameters() { return model->Parameters(); } + arma::mat& Parameters() { return model->Parameters(); } -// FFN* model; -// arma::mat input, target; -// } function; + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; -// REQUIRE(CheckGradient(function) <= 1e-4); -// } + REQUIRE(CheckGradient(function) <= 1e-4); +} /** * Simple linear no bias module test. @@ -689,13 +678,9 @@ TEST_CASE("GradientLinear3DLayerTest", "[ANNLayerTest]") TEST_CASE("SimpleLinearNoBiasLayerTest", "[ANNLayerTest]") { arma::mat output, input, delta; - LinearNoBias module(10); - arma::mat weights(10 * 10, 1); - module.InputDimensions() = std::vector({ 10 }); - module.ComputeOutputDimensions(); - module.SetWeights(weights.memptr()); - + LinearNoBias<> module(10, 10); module.Parameters().randu(); + module.Reset(); // Test the Forward function. input = arma::zeros(10, 1); @@ -703,7 +688,7 @@ TEST_CASE("SimpleLinearNoBiasLayerTest", "[ANNLayerTest]") REQUIRE(0 == arma::accu(output)); // Test the Backward function. - module.Backward(input, output, delta); + module.Backward(input, input, delta); REQUIRE(arma::accu(delta) == 0); } @@ -712,61 +697,39 @@ TEST_CASE("SimpleLinearNoBiasLayerTest", "[ANNLayerTest]") */ TEST_CASE("SimplePaddingLayerTest", "[ANNLayerTest]") { - arma::mat output, input, delta; - Padding module(1, 2, 3, 4); - module.InputDimensions() = std::vector({ 2, 5 }); - module.ComputeOutputDimensions(); + arma::mat output, input, delta, input1, output1; + Padding<> module(1, 2, 3, 4); // Test the Forward function. input = arma::randu(10, 1); - size_t totalOutputDimensions = module.OutputDimensions()[0]; - for (size_t i = 1; i < module.OutputDimensions().size(); ++i) - totalOutputDimensions *= module.OutputDimensions()[i]; - output.set_size(totalOutputDimensions, input.n_cols); - output.randu(); module.Forward(input, output); - REQUIRE(arma::accu(input) == Approx(arma::accu(output))); - REQUIRE(output.n_rows == (9 * 8)); // 2x5 --> 9x8 + REQUIRE(arma::accu(input) == arma::accu(output)); + REQUIRE(output.n_rows == input.n_rows + 3); + REQUIRE(output.n_cols == input.n_cols + 7); // Test the Backward function. - delta.set_size(input.n_rows, input.n_cols); module.Backward(input, output, delta); CheckMatrices(delta, input); // Test forward function for multiple filters. // Here it's 3 filters with height = 224, width = 224 // the output should be [226 * 226 * 3, 1] with 1 padding. - module = Padding(1, 1, 1, 1); - module.InputDimensions() = std::vector({ 224, 224, 3 }); - module.ComputeOutputDimensions(); - - input = arma::randu(224 * 224 * 3, 1); - totalOutputDimensions = module.OutputDimensions()[0]; - for (size_t i = 1; i < module.OutputDimensions().size(); ++i) - totalOutputDimensions *= module.OutputDimensions()[i]; - output.set_size(totalOutputDimensions, input.n_cols); - output.randu(); - module.Forward(input, output); - REQUIRE(arma::accu(input) == Approx(arma::accu(output))); - REQUIRE(output.n_rows == (226 * 226 * 3)); - REQUIRE(output.n_cols == 1); + Padding<> module1(1, 1, 1, 1, 224, 224); + input1 = arma::randu(224 * 224 * 3, 1); + module1.Forward(input1, output1); + REQUIRE(arma::accu(input1) == arma::accu(output1)); + REQUIRE(output1.n_rows == (226 * 226 * 3)); + REQUIRE(output1.n_cols == 1); // Test forward function for multiple batches with multiple filters. // Here it's 3 filters with height = 244, width = 244 // the output should be [246 * 246 * 3, 3] with 1 padding. - module.InputDimensions() = std::vector({ 244, 244, 3 }); - module.ComputeOutputDimensions(); - totalOutputDimensions = module.OutputDimensions()[0]; - for (size_t i = 1; i < module.OutputDimensions().size(); ++i) - totalOutputDimensions *= module.OutputDimensions()[i]; - - input = arma::randu(244 * 244 * 3, 3); - output.set_size(totalOutputDimensions, input.n_cols); - output.randu(); - module.Forward(input, output); - REQUIRE(output.n_rows == (246 * 246 * 3)); - REQUIRE(output.n_cols == 3); - REQUIRE(arma::accu(input) == Approx(arma::accu(output))); + Padding<> module2(1, 1, 1, 1, 244, 244); + input1 = arma::randu(244 * 244 * 3, 3); + module2.Forward(input1, output1); + REQUIRE(arma::accu(input1) == arma::accu(output1)); + REQUIRE(output1.n_rows == (246 * 246 * 3)); + REQUIRE(output1.n_cols == 3); } /** @@ -782,12 +745,7 @@ TEST_CASE("JacobianLinearNoBiasLayerTest", "[ANNLayerTest]") arma::mat input; input.set_size(inputElements, 1); - LinearNoBias module(outputElements); - arma::mat weights(inputElements * outputElements, 1); - module.InputDimensions() = std::vector({ inputElements }); - module.ComputeOutputDimensions(); - module.SetWeights(weights.memptr()); - + LinearNoBias<> module(inputElements, outputElements); module.Parameters().randu(); double error = JacobianTest(module, input); @@ -807,11 +765,13 @@ TEST_CASE("GradientLinearNoBiasLayerTest", "[ANNLayerTest]") input(arma::randu(10, 1)), target(arma::mat("0")) { - model = new FFN(); - model->ResetData(input, target); - model->Add(10); - model->Add(2); - model->Add(); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 10); + model->Add >(10, 2); + model->Add >(); } ~GradientFunction() @@ -828,37 +788,37 @@ TEST_CASE("GradientLinearNoBiasLayerTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, NguyenWidrowInitialization>* model; arma::mat input, target; } function; REQUIRE(CheckGradient(function) <= 1e-4); } -// /** -// * Jacobian negative log likelihood module test. -// */ -// TEST_CASE("JacobianNegativeLogLikelihoodLayerTest", "[ANNLayerTest]") -// { -// for (size_t i = 0; i < 5; ++i) -// { -// NegativeLogLikelihood module; -// const size_t inputElements = math::RandInt(5, 100); -// arma::mat input; -// RandomInitialization init(0, 1); -// init.Initialize(input, inputElements, 1); +/** + * Jacobian negative log likelihood module test. + */ +TEST_CASE("JacobianNegativeLogLikelihoodLayerTest", "[ANNLayerTest]") +{ + for (size_t i = 0; i < 5; ++i) + { + NegativeLogLikelihood<> module; + const size_t inputElements = math::RandInt(5, 100); + arma::mat input; + RandomInitialization init(0, 1); + init.Initialize(input, inputElements, 1); -// arma::mat target(1, 1); -// target(0) = math::RandInt(0, inputElements - 2); + arma::mat target(1, 1); + target(0) = math::RandInt(0, inputElements - 2); -// double error = JacobianPerformanceTest(module, input, target); -// REQUIRE(error <= 1e-5); -// } -// } + double error = JacobianPerformanceTest(module, input, target); + REQUIRE(error <= 1e-5); + } +} /** * Jacobian LeakyReLU module test. - * + */ TEST_CASE("JacobianLeakyReLULayerTest", "[ANNLayerTest]") { for (size_t i = 0; i < 5; ++i) @@ -868,17 +828,16 @@ TEST_CASE("JacobianLeakyReLULayerTest", "[ANNLayerTest]") arma::mat input; input.set_size(inputElements, 1); - LeakyReLU module; + LeakyReLU<> module; double error = JacobianTest(module, input); REQUIRE(error <= 1e-5); } } -*/ /** * Jacobian FlexibleReLU module test. - * + */ TEST_CASE("JacobianFlexibleReLULayerTest", "[ANNLayerTest]") { for (size_t i = 0; i < 5; ++i) @@ -888,17 +847,16 @@ TEST_CASE("JacobianFlexibleReLULayerTest", "[ANNLayerTest]") arma::mat input; input.set_size(inputElements, 1); - FlexibleReLU module; + FlexibleReLU<> module; double error = JacobianTest(module, input); REQUIRE(error <= 1e-5); } } -*/ /** * Flexible ReLU layer numerical gradient test. - * + */ TEST_CASE("GradientFlexibleReLULayerTest", "[ANNLayerTest]") { // Add function gradient instantiation. @@ -908,14 +866,15 @@ TEST_CASE("GradientFlexibleReLULayerTest", "[ANNLayerTest]") input(arma::randu(2, 1)), target(arma::mat("0")) { - model = new FFN( - NegativeLogLikelihood(), RandomInitialization(0.1, 0.5)); + model = new FFN, RandomInitialization>( + NegativeLogLikelihood<>(), RandomInitialization(0.1, 0.5)); - model->ResetData(input, target); - model->Add(2, 2); - model->Add(2, 5); - model->Add(0.05); - model->Add(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(2, 2); + model->Add >(2, 5); + model->Add >(0.05); + model->Add >(); } ~GradientFunction() @@ -932,17 +891,16 @@ TEST_CASE("GradientFlexibleReLULayerTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, RandomInitialization>* model; arma::mat input, target; } function; REQUIRE(CheckGradient(function) <= 1e-4); } -*/ /** * Jacobian MultiplyConstant module test. - * + */ TEST_CASE("JacobianMultiplyConstantLayerTest", "[ANNLayerTest]") { for (size_t i = 0; i < 5; ++i) @@ -952,50 +910,49 @@ TEST_CASE("JacobianMultiplyConstantLayerTest", "[ANNLayerTest]") arma::mat input; input.set_size(inputElements, 1); - MultiplyConstant module(3.0); + MultiplyConstant<> module(3.0); double error = JacobianTest(module, input); REQUIRE(error <= 1e-5); } } -*/ /** * Check whether copying and moving network with MultiplyConstant is working or * not. */ -// TEST_CASE("CheckCopyMoveMultiplyConstantTest", "[ANNLayerTest]") -// { -// arma::mat input(2, 1000); -// input.randu(); -// -// arma::mat output1; -// arma::mat output2; -// arma::mat output3; -// arma::mat output4; -// -// MultiplyConstant<> *module1 = new MultiplyConstant<>(3.0); -// module1->Forward(input, output1); -// -// MultiplyConstant<> module2 = *module1; -// delete module1; -// -// module2.Forward(input, output2); -// CheckMatrices(output1, output2); -// -// MultiplyConstant<> *module3 = new MultiplyConstant<>(3.0); -// module3->Forward(input, output3); -// -// MultiplyConstant<> module4(std::move(*module3)); -// delete module3; -// -// module4.Forward(input, output4); -// CheckMatrices(output3, output4); -// } +TEST_CASE("CheckCopyMoveMultiplyConstantTest", "[ANNLayerTest]") +{ + arma::mat input(2, 1000); + input.randu(); + + arma::mat output1; + arma::mat output2; + arma::mat output3; + arma::mat output4; + + MultiplyConstant<> *module1 = new MultiplyConstant<>(3.0); + module1->Forward(input, output1); + + MultiplyConstant<> module2 = *module1; + delete module1; + + module2.Forward(input, output2); + CheckMatrices(output1, output2); + + MultiplyConstant<> *module3 = new MultiplyConstant<>(3.0); + module3->Forward(input, output3); + + MultiplyConstant<> module4(std::move(*module3)); + delete module3; + + module4.Forward(input, output4); + CheckMatrices(output3, output4); +} /** * Jacobian HardTanH module test. - * + */ TEST_CASE("JacobianHardTanHLayerTest", "[ANNLayerTest]") { for (size_t i = 0; i < 5; ++i) @@ -1005,20 +962,18 @@ TEST_CASE("JacobianHardTanHLayerTest", "[ANNLayerTest]") arma::mat input; input.set_size(inputElements, 1); - HardTanH module; + HardTanH<> module; double error = JacobianTest(module, input); REQUIRE(error <= 1e-5); } } -*/ /** * Simple select module test. - * + */ TEST_CASE("SimpleSelectLayerTest", "[ANNLayerTest]") { - // TODO: this needs to be adapted arma::mat outputA, outputB, input, delta; input = arma::ones(10, 5); @@ -1028,12 +983,12 @@ TEST_CASE("SimpleSelectLayerTest", "[ANNLayerTest]") } // Test the Forward function. - Select moduleA(3); + Select<> moduleA(3); moduleA.Forward(input, outputA); REQUIRE(30 == arma::accu(outputA)); // Test the Forward function. - Select moduleB(3, 5); + Select<> moduleB(3, 5); moduleB.Forward(input, outputB); REQUIRE(15 == arma::accu(outputB)); @@ -1045,33 +1000,31 @@ TEST_CASE("SimpleSelectLayerTest", "[ANNLayerTest]") moduleB.Backward(input, outputA, delta); REQUIRE(15 == arma::accu(delta)); } -*/ /** * Test that the functions that can access the parameters of the * Select layer work. - * + */ TEST_CASE("SelectLayerParametersTest", "[ANNLayerTest]") { // Parameter order : index, elements. - Select layer(3, 5); + Select<> layer(3, 5); // Make sure we can get the parameters successfully. REQUIRE(layer.Index() == 3); REQUIRE(layer.NumElements() == 5); } -*/ /** * Simple join module test. - * + */ TEST_CASE("SimpleJoinLayerTest", "[ANNLayerTest]") { arma::mat output, input, delta; input = arma::ones(10, 5); // Test the Forward function. - Join module; + Join<> module; module.Forward(input, output); REQUIRE(50 == arma::accu(output)); @@ -1085,1234 +1038,635 @@ TEST_CASE("SimpleJoinLayerTest", "[ANNLayerTest]") b = delta.n_rows == input.n_rows && input.n_cols; REQUIRE(b == true); } -*/ - -// /** -// * Simple add merge module test. -// */ -// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// input = arma::ones(10, 1); - -// for (size_t i = 0; i < 5; ++i) -// { -// AddMerge<> module(false, false); -// const size_t numMergeModules = math::RandInt(2, 10); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); - -// module.Add >(identityLayer); -// } - -// // Test the Forward function. -// module.Forward(input, output); -// REQUIRE(10 * numMergeModules == arma::accu(output)); - -// // Test the Backward function. -// module.Backward(input, output, delta); -// REQUIRE(arma::accu(output) == arma::accu(delta)); -// } -// } - -// /** -// * Test the LSTM layer with a user defined rho parameter and without. -// */ -// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// RandomInitialization init(0.5, 0.5); - -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); - -// // Use LSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); - -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); - -// // Use LSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); - -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); - -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -// /** -// * LSTM layer numerical gradient test. -// */ -// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") -// { -// // LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::ones(1, 1, 5)) -// { -// const size_t rho = 5; - -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// RNN* model; -// arma::cube input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * Test that the functions that can modify and access the parameters of the -// * LSTM layer work. -// */ -// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// LSTM<> layer1(1, 2, 3); -// LSTM<> layer2(1, 2, 4); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); - -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; - -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -// /** -// * Test the FastLSTM layer with a user defined rho parameter and without. -// */ -// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// RandomInitialization init(0.5, 0.5); - -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); - -// // Use FastLSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); - -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); - -// // Use FastLSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); - -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); - -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } - -// /** -// * FastLSTM layer numerical gradient test. -// */ -// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") -// { -// // Fast LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::ones(1, 1, 5)) -// { -// const size_t rho = 5; - -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// RNN* model; -// arma::cube input, target; -// } function; - -// // The threshold should be << 0.1 but since the Fast LSTM layer uses an -// // approximation of the sigmoid function the estimated gradient is not -// // correct. -// REQUIRE(CheckGradient(function) <= 0.2); -// } - -// /** -// * Test that the functions that can modify and access the parameters of the -// * Fast LSTM layer work. -// */ -// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// FastLSTM<> layer1(1, 2, 3); -// FastLSTM<> layer2(1, 2, 4); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); - -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; - -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -// /** -// * Check whether copying and moving network with FastLSTM is working or not. -// */ -// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; - -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); - -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); - -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); - -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -// /** -// * Check whether copying and moving network with LSTM is working or not. -// */ -// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; - -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); - -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); - -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); - -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } - -// /** -// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell -// * state. Besides output, the overloaded function provides read access to cell -// * state of the LSTM layer. -// */ -// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; - -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); - -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; - -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); - -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); - -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); - -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); - -// // Apply Forward() on LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// false); // Don't write into the cell state. - -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); - -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; - -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); - -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } -// } - -// /** -// * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell -// * state. Besides output, the overloaded function provides write access to cell -// * state of the LSTM layer. -// */ -// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; - -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); - -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; -// arma::mat cellCalc; - -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); - -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); - -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); - -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); - -// if (cellLstm.is_empty()) -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(outputSize, input.n_cols); -// cellCalc = arma::zeros(outputSize, input.n_cols); -// } -// else -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); -// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); -// } - -// // Apply Forward() on the LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. - -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); - -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; - -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); - -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); - -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } - -// // Attempting to write empty matrix into cell state. -// lstm.Reset(); -// lstm.ResetCell(rho); -// arma::mat stepData(input.slice(0).memptr(), -// input.n_rows, input.n_cols, false, true); - -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. - -// for (size_t seqNum = 1; seqNum < rho; ++seqNum) -// { -// arma::mat empty; -// // Should throw error. -// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. -// outLstm, // Output. -// empty, // Cell state. -// true), // Write into cell state. -// std::runtime_error); -// } -// } - -// /** -// * Test that the functions that can modify and access the parameters of the -// * GRU layer work. -// */ -// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// GRU<> layer1(1, 2, 3); -// GRU<> layer2(1, 2, 4); - -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); - -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; - -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } - -// /** -// * Check if the gradients computed by GRU cell are close enough to the -// * approximation of the gradients. -// */ -// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") -// { -// // GRU function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::ones(1, 1, 5)) -// { -// const size_t rho = 5; - -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// arma::mat output; -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// RNN* model; -// arma::cube input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -// /** -// * GRU layer manual forward test. -// */ -// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") -// { -// // This will make it easier to clean memory later. -// GRU<>* gruAlloc = new GRU<>(3, 3, 5); -// GRU<>& gru = *gruAlloc; - -// // Initialize the weights to all ones. -// NetworkInitialization -// networkInit(ConstInitialization(1)); -// networkInit.Initialize(gru.Model(), gru.Parameters()); - -// // Provide input of all ones. -// arma::mat input = arma::ones(3, 1); -// arma::mat output; - -// gru.Forward(input, output); - -// // Compute the z_t gate output. -// arma::mat expectedOutput = arma::ones(3, 1); -// expectedOutput *= -4; -// expectedOutput = arma::exp(expectedOutput); -// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); -// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; - -// // For the first input the output should be equal to the output of -// // gate z_t as the previous output fed to the cell is all zeros. -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); - -// expectedOutput = output; - -// gru.Forward(input, output); - -// double s = arma::as_scalar(arma::sum(expectedOutput)); - -// // Compute the value of z_t gate for the second input. -// arma::mat z_t = arma::ones(3, 1); -// z_t *= -(s + 4); -// z_t = arma::exp(z_t); -// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); - -// // Compute the value of o_t gate for the second input. -// arma::mat o_t = arma::ones(3, 1); -// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); -// o_t = arma::exp(o_t); -// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); - -// // Expected output for the second input. -// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; - -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); - -// LayerTypes<> layer(gruAlloc); -// boost::apply_visitor(DeleteVisitor(), layer); -// } /** * Simple add merge module test. */ -// TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; -// input = arma::ones(10, 1); -// -// for (size_t i = 0; i < 5; ++i) -// { -// AddMerge<> module(false, false); -// const size_t numMergeModules = math::RandInt(2, 10); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); -// -// module.Add >(identityLayer); -// } -// -// // Test the Forward function. -// module.Forward(input, output); -// REQUIRE(10 * numMergeModules == arma::accu(output)); -// -// // Test the Backward function. -// module.Backward(input, output, delta); -// REQUIRE(arma::accu(output) == arma::accu(delta)); -// } -// } +TEST_CASE("SimpleAddMergeLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + input = arma::ones(10, 1); + + for (size_t i = 0; i < 5; ++i) + { + AddMerge<> module(false, false); + const size_t numMergeModules = math::RandInt(2, 10); + for (size_t m = 0; m < numMergeModules; ++m) + { + IdentityLayer<> identityLayer; + identityLayer.Forward(input, identityLayer.OutputParameter()); + + module.Add >(identityLayer); + } + + // Test the Forward function. + module.Forward(input, output); + REQUIRE(10 * numMergeModules == arma::accu(output)); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(output) == arma::accu(delta)); + } +} /** * Test the LSTM layer with a user defined rho parameter and without. */ -// TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::zeros(1, 1, 5); -// RandomInitialization init(0.5, 0.5); -// -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); -// -// // Use LSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); -// -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); -// -// // Use LSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); -// -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); -// -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } +TEST_CASE("LSTMRrhoTest", "[ANNLayerTest]") +{ + const size_t rho = 5; + arma::cube input = arma::randu(1, 1, 5); + arma::cube target = arma::zeros(1, 1, 5); + RandomInitialization init(0.5, 0.5); + + // Create model with user defined rho parameter. + RNN, RandomInitialization> modelA( + rho, false, NegativeLogLikelihood<>(), init); + modelA.Add >(); + modelA.Add >(1, 10); + + // Use LSTM layer with rho. + modelA.Add >(10, 3, rho); + modelA.Add >(); + + // Create model without user defined rho parameter. + RNN > modelB( + rho, false, NegativeLogLikelihood<>(), init); + modelB.Add >(); + modelB.Add >(1, 10); + + // Use LSTM layer with rho = MAXSIZE. + modelB.Add >(10, 3); + modelB.Add >(); + + ens::StandardSGD opt(0.1, 1, 5, -100, false); + modelA.Train(input, target, opt); + modelB.Train(input, target, opt); + + CheckMatrices(modelB.Parameters(), modelA.Parameters()); +} /** * LSTM layer numerical gradient test. */ -// TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") -// { -// // LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::zeros(1, 1, 5)) -// { -// const size_t rho = 5; -// -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// RNN* model; -// arma::cube input, target; -// } function; -// -// REQUIRE(CheckGradient(function) <= 1e-4); -// } +TEST_CASE("GradientLSTMLayerTest", "[ANNLayerTest]") +{ + // LSTM function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(1, 1, 5)), + target(arma::zeros(1, 1, 5)) + { + const size_t rho = 5; + + model = new RNN >(rho); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(1, 10); + model->Add >(10, 3, rho); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + RNN >* model; + arma::cube input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} /** * Test that the functions that can modify and access the parameters of the * LSTM layer work. */ -// TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// LSTM<> layer1(1, 2, 3); -// LSTM<> layer2(1, 2, 4); -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); -// -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; -// -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } +TEST_CASE("LSTMLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inSize, outSize, rho. + LSTM<> layer1(1, 2, 3); + LSTM<> layer2(1, 2, 4); + + // Make sure we can get the parameters successfully. + REQUIRE(layer1.InSize() == 1); + REQUIRE(layer1.OutSize() == 2); + REQUIRE(layer1.Rho() == 3); + + // Now modify the parameters to match the second layer. + layer1.Rho() = 4; + + // Now ensure all the results are the same. + REQUIRE(layer1.InSize() == layer2.InSize()); + REQUIRE(layer1.OutSize() == layer2.OutSize()); + REQUIRE(layer1.Rho() == layer2.Rho()); +} /** * Test the FastLSTM layer with a user defined rho parameter and without. */ -// TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") -// { -// const size_t rho = 5; -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::zeros(1, 1, 5); -// RandomInitialization init(0.5, 0.5); -// -// // Create model with user defined rho parameter. -// RNN modelA( -// rho, false, NegativeLogLikelihood(), init); -// modelA.Add >(); -// modelA.Add >(1, 10); -// -// // Use FastLSTM layer with rho. -// modelA.Add >(10, 3, rho); -// modelA.Add >(); -// -// // Create model without user defined rho parameter. -// RNN modelB( -// rho, false, NegativeLogLikelihood(), init); -// modelB.Add >(); -// modelB.Add >(1, 10); -// -// // Use FastLSTM layer with rho = MAXSIZE. -// modelB.Add >(10, 3); -// modelB.Add >(); -// -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// modelA.Train(input, target, opt); -// modelB.Train(input, target, opt); -// -// CheckMatrices(modelB.Parameters(), modelA.Parameters()); -// } +TEST_CASE("FastLSTMRrhoTest", "[ANNLayerTest]") +{ + const size_t rho = 5; + arma::cube input = arma::randu(1, 1, 5); + arma::cube target = arma::zeros(1, 1, 5); + RandomInitialization init(0.5, 0.5); + + // Create model with user defined rho parameter. + RNN, RandomInitialization> modelA( + rho, false, NegativeLogLikelihood<>(), init); + modelA.Add >(); + modelA.Add >(1, 10); + + // Use FastLSTM layer with rho. + modelA.Add >(10, 3, rho); + modelA.Add >(); + + // Create model without user defined rho parameter. + RNN > modelB( + rho, false, NegativeLogLikelihood<>(), init); + modelB.Add >(); + modelB.Add >(1, 10); + + // Use FastLSTM layer with rho = MAXSIZE. + modelB.Add >(10, 3); + modelB.Add >(); + + ens::StandardSGD opt(0.1, 1, 5, -100, false); + modelA.Train(input, target, opt); + modelB.Train(input, target, opt); + + CheckMatrices(modelB.Parameters(), modelA.Parameters()); +} /** * FastLSTM layer numerical gradient test. */ -// TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") -// { -// // Fast LSTM function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::zeros(1, 1, 5)) -// { -// const size_t rho = 5; -// -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// RNN* model; -// arma::cube input, target; -// } function; -// -// // The threshold should be << 0.1 but since the Fast LSTM layer uses an -// // approximation of the sigmoid function the estimated gradient is not -// // correct. -// REQUIRE(CheckGradient(function) <= 0.2); -// } +TEST_CASE("GradientFastLSTMLayerTest", "[ANNLayerTest]") +{ + // Fast LSTM function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(1, 1, 5)), + target(arma::zeros(1, 1, 5)) + { + const size_t rho = 5; + + model = new RNN >(rho); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(1, 10); + model->Add >(10, 3, rho); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + RNN >* model; + arma::cube input, target; + } function; + + // The threshold should be << 0.1 but since the Fast LSTM layer uses an + // approximation of the sigmoid function the estimated gradient is not + // correct. + REQUIRE(CheckGradient(function) <= 0.2); +} /** * Test that the functions that can modify and access the parameters of the * Fast LSTM layer work. */ -// TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// FastLSTM<> layer1(1, 2, 3); -// FastLSTM<> layer2(1, 2, 4); -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); -// -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; -// -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } +TEST_CASE("FastLSTMLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inSize, outSize, rho. + FastLSTM<> layer1(1, 2, 3); + FastLSTM<> layer2(1, 2, 4); + + // Make sure we can get the parameters successfully. + REQUIRE(layer1.InSize() == 1); + REQUIRE(layer1.OutSize() == 2); + REQUIRE(layer1.Rho() == 3); + + // Now modify the parameters to match the second layer. + layer1.Rho() = 4; + + // Now ensure all the results are the same. + REQUIRE(layer1.InSize() == layer2.InSize()); + REQUIRE(layer1.OutSize() == layer2.OutSize()); + REQUIRE(layer1.Rho() == layer2.Rho()); +} /** * Check whether copying and moving network with FastLSTM is working or not. */ -// TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; -// -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); -// -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); -// -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } +TEST_CASE("CheckCopyMoveFastLSTMTest", "[ANNLayerTest]") +{ + arma::cube input = arma::randu(1, 1, 5); + arma::cube target = arma::ones(1, 1, 5); + const size_t rho = 5; + + RNN > *model1 = + new RNN >(rho); + model1->Predictors() = input; + model1->Responses() = target; + model1->Add >(); + model1->Add >(1, 10); + model1->Add >(10, 3, rho); + model1->Add >(); + + RNN > *model2 = + new RNN >(rho); + model2->Predictors() = input; + model2->Responses() = target; + model2->Add >(); + model2->Add >(1, 10); + model2->Add >(10, 3, rho); + model2->Add >(); + + // Check whether copy constructor is working or not. + CheckRNNCopyFunction<>(model1, input, target, 1); + + // Check whether move constructor is working or not. + CheckRNNMoveFunction<>(model2, input, target, 1); +} /** * Check whether copying and moving network with LSTM is working or not. */ -// TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") -// { -// arma::cube input = arma::randu(1, 1, 5); -// arma::cube target = arma::ones(1, 1, 5); -// const size_t rho = 5; -// -// RNN *model1 = -// new RNN(rho); -// model1->ResetData(input, target); -// model1->Add >(); -// model1->Add >(1, 10); -// model1->Add >(10, 3, rho); -// model1->Add >(); -// -// RNN *model2 = -// new RNN(rho); -// model2->ResetData(input, target); -// model2->Add >(); -// model2->Add >(1, 10); -// model2->Add >(10, 3, rho); -// model2->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckRNNCopyFunction<>(model1, input, target, 1); -// -// // Check whether move constructor is working or not. -// CheckRNNMoveFunction<>(model2, input, target, 1); -// } +TEST_CASE("CheckCopyMoveLSTMTest", "[ANNLayerTest]") +{ + arma::cube input = arma::randu(1, 1, 5); + arma::cube target = arma::ones(1, 1, 5); + const size_t rho = 5; + + RNN > *model1 = + new RNN >(rho); + model1->Predictors() = input; + model1->Responses() = target; + model1->Add >(); + model1->Add >(1, 10); + model1->Add >(10, 3, rho); + model1->Add >(); + + RNN > *model2 = + new RNN >(rho); + model2->Predictors() = input; + model2->Responses() = target; + model2->Add >(); + model2->Add >(1, 10); + model2->Add >(10, 3, rho); + model2->Add >(); + + // Check whether copy constructor is working or not. + CheckRNNCopyFunction<>(model1, input, target, 1); + + // Check whether move constructor is working or not. + CheckRNNMoveFunction<>(model2, input, target, 1); +} /** * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell * state. Besides output, the overloaded function provides read access to cell * state of the LSTM layer. */ -// TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; -// -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); -// -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; -// -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); -// -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); -// -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); -// -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); -// -// // Apply Forward() on LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// false); // Don't write into the cell state. -// -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); -// -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; -// -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); -// -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } -// } +TEST_CASE("ReadCellStateParamLSTMLayerTest", "[ANNLayerTest]") +{ + const size_t rho = 5, inputSize = 3, outputSize = 2; + + // Provide input of all ones. + arma::cube input = arma::ones(inputSize, outputSize, rho); + + arma::mat inputGate, forgetGate, outputGate, hidden; + arma::mat outLstm, cellLstm; + + // LSTM layer. + LSTM<> lstm(inputSize, outputSize, rho); + lstm.Reset(); + lstm.ResetCell(rho); + + // Initialize the weights to all ones. + lstm.Parameters().ones(); + + arma::mat inputWeight = arma::ones(outputSize, inputSize); + arma::mat outputWeight = arma::ones(outputSize, outputSize); + arma::mat bias = arma::ones(outputSize, input.n_cols); + arma::mat cellCalc = arma::zeros(outputSize, input.n_cols); + arma::mat outCalc = arma::zeros(outputSize, input.n_cols); + + for (size_t seqNum = 0; seqNum < rho; ++seqNum) + { + // Wrap a matrix around our data to avoid a copy. + arma::mat stepData(input.slice(seqNum).memptr(), + input.n_rows, input.n_cols, false, true); + + // Apply Forward() on LSTM layer. + lstm.Forward(stepData, // Input. + outLstm, // Output. + cellLstm, // Cell state. + false); // Don't write into the cell state. + + // Compute the value of cell state and output. + // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). + inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + + outputWeight * outCalc + outputWeight % cellCalc + bias))); + + // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). + forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + + outputWeight * outCalc + outputWeight % cellCalc + bias))); + + // z = tanh(W.dot(x) + W.dot(h) + b). + hidden = arma::tanh(inputWeight * stepData + + outputWeight * outCalc + bias); + + // c = f * c + i * z. + cellCalc = forgetGate % cellCalc + inputGate % hidden; + + // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). + outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + + outputWeight * outCalc + outputWeight % cellCalc + bias))); + + // h = o * tanh(c). + outCalc = outputGate % arma::tanh(cellCalc); + + CheckMatrices(outLstm, outCalc, 1e-12); + CheckMatrices(cellLstm, cellCalc, 1e-12); + } +} /** * Testing the overloaded Forward() of the LSTM layer, for retrieving the cell * state. Besides output, the overloaded function provides write access to cell * state of the LSTM layer. */ -// TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") -// { -// const size_t rho = 5, inputSize = 3, outputSize = 2; -// -// // Provide input of all ones. -// arma::cube input = arma::ones(inputSize, outputSize, rho); -// -// arma::mat inputGate, forgetGate, outputGate, hidden; -// arma::mat outLstm, cellLstm; -// arma::mat cellCalc; -// -// // LSTM layer. -// LSTM<> lstm(inputSize, outputSize, rho); -// lstm.Reset(); -// lstm.ResetCell(rho); -// -// // Initialize the weights to all ones. -// lstm.Parameters().ones(); -// -// arma::mat inputWeight = arma::ones(outputSize, inputSize); -// arma::mat outputWeight = arma::ones(outputSize, outputSize); -// arma::mat bias = arma::ones(outputSize, input.n_cols); -// arma::mat outCalc = arma::zeros(outputSize, input.n_cols); -// -// for (size_t seqNum = 0; seqNum < rho; ++seqNum) -// { -// // Wrap a matrix around our data to avoid a copy. -// arma::mat stepData(input.slice(seqNum).memptr(), -// input.n_rows, input.n_cols, false, true); -// -// if (cellLstm.is_empty()) -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(outputSize, input.n_cols); -// cellCalc = arma::zeros(outputSize, input.n_cols); -// } -// else -// { -// // Set the cell state to zeros. -// cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); -// cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); -// } -// -// // Apply Forward() on the LSTM layer. -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. -// -// // Compute the value of cell state and output. -// // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // z = tanh(W.dot(x) + W.dot(h) + b). -// hidden = arma::tanh(inputWeight * stepData + -// outputWeight * outCalc + bias); -// -// // c = f * c + i * z. -// cellCalc = forgetGate % cellCalc + inputGate % hidden; -// -// // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). -// outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + -// outputWeight * outCalc + outputWeight % cellCalc + bias))); -// -// // h = o * tanh(c). -// outCalc = outputGate % arma::tanh(cellCalc); -// -// CheckMatrices(outLstm, outCalc, 1e-12); -// CheckMatrices(cellLstm, cellCalc, 1e-12); -// } -// -// // Attempting to write empty matrix into cell state. -// lstm.Reset(); -// lstm.ResetCell(rho); -// arma::mat stepData(input.slice(0).memptr(), -// input.n_rows, input.n_cols, false, true); -// -// lstm.Forward(stepData, // Input. -// outLstm, // Output. -// cellLstm, // Cell state. -// true); // Write into cell state. -// -// for (size_t seqNum = 1; seqNum < rho; ++seqNum) -// { -// arma::mat empty; -// // Should throw error. -// REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. -// outLstm, // Output. -// empty, // Cell state. -// true), // Write into cell state. -// std::runtime_error); -// } -// } +TEST_CASE("WriteCellStateParamLSTMLayerTest", "[ANNLayerTest]") +{ + const size_t rho = 5, inputSize = 3, outputSize = 2; + + // Provide input of all ones. + arma::cube input = arma::ones(inputSize, outputSize, rho); + + arma::mat inputGate, forgetGate, outputGate, hidden; + arma::mat outLstm, cellLstm; + arma::mat cellCalc; + + // LSTM layer. + LSTM<> lstm(inputSize, outputSize, rho); + lstm.Reset(); + lstm.ResetCell(rho); + + // Initialize the weights to all ones. + lstm.Parameters().ones(); + + arma::mat inputWeight = arma::ones(outputSize, inputSize); + arma::mat outputWeight = arma::ones(outputSize, outputSize); + arma::mat bias = arma::ones(outputSize, input.n_cols); + arma::mat outCalc = arma::zeros(outputSize, input.n_cols); + + for (size_t seqNum = 0; seqNum < rho; ++seqNum) + { + // Wrap a matrix around our data to avoid a copy. + arma::mat stepData(input.slice(seqNum).memptr(), + input.n_rows, input.n_cols, false, true); + + if (cellLstm.is_empty()) + { + // Set the cell state to zeros. + cellLstm = arma::zeros(outputSize, input.n_cols); + cellCalc = arma::zeros(outputSize, input.n_cols); + } + else + { + // Set the cell state to zeros. + cellLstm = arma::zeros(cellLstm.n_rows, cellLstm.n_cols); + cellCalc = arma::zeros(cellCalc.n_rows, cellCalc.n_cols); + } + + // Apply Forward() on the LSTM layer. + lstm.Forward(stepData, // Input. + outLstm, // Output. + cellLstm, // Cell state. + true); // Write into cell state. + + // Compute the value of cell state and output. + // i = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). + inputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + + outputWeight * outCalc + outputWeight % cellCalc + bias))); + + // f = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). + forgetGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + + outputWeight * outCalc + outputWeight % cellCalc + bias))); + + // z = tanh(W.dot(x) + W.dot(h) + b). + hidden = arma::tanh(inputWeight * stepData + + outputWeight * outCalc + bias); + + // c = f * c + i * z. + cellCalc = forgetGate % cellCalc + inputGate % hidden; + + // o = sigmoid(W.dot(x) + W.dot(h) + W.dot(c) + b). + outputGate = 1.0 /(1 + arma::exp(-(inputWeight * stepData + + outputWeight * outCalc + outputWeight % cellCalc + bias))); + + // h = o * tanh(c). + outCalc = outputGate % arma::tanh(cellCalc); + + CheckMatrices(outLstm, outCalc, 1e-12); + CheckMatrices(cellLstm, cellCalc, 1e-12); + } + + // Attempting to write empty matrix into cell state. + lstm.Reset(); + lstm.ResetCell(rho); + arma::mat stepData(input.slice(0).memptr(), + input.n_rows, input.n_cols, false, true); + + lstm.Forward(stepData, // Input. + outLstm, // Output. + cellLstm, // Cell state. + true); // Write into cell state. + + for (size_t seqNum = 1; seqNum < rho; ++seqNum) + { + arma::mat empty; + // Should throw error. + REQUIRE_THROWS_AS(lstm.Forward(stepData, // Input. + outLstm, // Output. + empty, // Cell state. + true), // Write into cell state. + std::runtime_error); + } +} /** * Test that the functions that can modify and access the parameters of the * GRU layer work. */ -// TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : inSize, outSize, rho. -// GRU<> layer1(1, 2, 3); -// GRU<> layer2(1, 2, 4); -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InSize() == 1); -// REQUIRE(layer1.OutSize() == 2); -// REQUIRE(layer1.Rho() == 3); -// -// // Now modify the parameters to match the second layer. -// layer1.Rho() = 4; -// -// // Now ensure all the results are the same. -// REQUIRE(layer1.InSize() == layer2.InSize()); -// REQUIRE(layer1.OutSize() == layer2.OutSize()); -// REQUIRE(layer1.Rho() == layer2.Rho()); -// } +TEST_CASE("GRULayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : inSize, outSize, rho. + GRU<> layer1(1, 2, 3); + GRU<> layer2(1, 2, 4); + + // Make sure we can get the parameters successfully. + REQUIRE(layer1.InSize() == 1); + REQUIRE(layer1.OutSize() == 2); + REQUIRE(layer1.Rho() == 3); + + // Now modify the parameters to match the second layer. + layer1.Rho() = 4; + + // Now ensure all the results are the same. + REQUIRE(layer1.InSize() == layer2.InSize()); + REQUIRE(layer1.OutSize() == layer2.OutSize()); + REQUIRE(layer1.Rho() == layer2.Rho()); +} /** * Check if the gradients computed by GRU cell are close enough to the * approximation of the gradients. */ -// TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") -// { -// // GRU function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(1, 1, 5)), -// target(arma::zeros(1, 1, 5)) -// { -// const size_t rho = 5; -// -// model = new RNN(rho); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 10); -// model->Add >(10, 3, rho); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// arma::mat output; -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// RNN* model; -// arma::cube input, target; -// } function; -// -// REQUIRE(CheckGradient(function) <= 1e-4); -// } +TEST_CASE("GradientGRULayerTest", "[ANNLayerTest]") +{ + // GRU function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(1, 1, 5)), + target(arma::zeros(1, 1, 5)) + { + const size_t rho = 5; + + model = new RNN >(rho); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(1, 10); + model->Add >(10, 3, rho); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + arma::mat output; + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + RNN >* model; + arma::cube input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} /** * GRU layer manual forward test. */ -// TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") -// { -// // This will make it easier to clean memory later. -// GRU<>* gruAlloc = new GRU<>(3, 3, 5); -// GRU<>& gru = *gruAlloc; -// -// // Initialize the weights to all ones. -// NetworkInitialization -// networkInit(ConstInitialization(1)); -// networkInit.Initialize(gru.Model(), gru.Parameters()); -// -// // Provide input of all ones. -// arma::mat input = arma::ones(3, 1); -// arma::mat output; -// -// gru.Forward(input, output); -// -// // Compute the z_t gate output. -// arma::mat expectedOutput = arma::ones(3, 1); -// expectedOutput *= -4; -// expectedOutput = arma::exp(expectedOutput); -// expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); -// expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; -// -// // For the first input the output should be equal to the output of -// // gate z_t as the previous output fed to the cell is all zeros. -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); -// -// expectedOutput = output; -// -// gru.Forward(input, output); -// -// double s = arma::as_scalar(arma::sum(expectedOutput)); -// -// // Compute the value of z_t gate for the second input. -// arma::mat z_t = arma::ones(3, 1); -// z_t *= -(s + 4); -// z_t = arma::exp(z_t); -// z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); -// -// // Compute the value of o_t gate for the second input. -// arma::mat o_t = arma::ones(3, 1); -// o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); -// o_t = arma::exp(o_t); -// o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); -// -// // Expected output for the second input. -// expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; -// -// REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); -// -// LayerTypes<> layer(gruAlloc); -// boost::apply_visitor(DeleteVisitor(), layer); -// } +TEST_CASE("ForwardGRULayerTest", "[ANNLayerTest]") +{ + // This will make it easier to clean memory later. + GRU<>* gruAlloc = new GRU<>(3, 3, 5); + GRU<>& gru = *gruAlloc; + + // Initialize the weights to all ones. + NetworkInitialization + networkInit(ConstInitialization(1)); + networkInit.Initialize(gru.Model(), gru.Parameters()); + + // Provide input of all ones. + arma::mat input = arma::ones(3, 1); + arma::mat output; + + gru.Forward(input, output); + + // Compute the z_t gate output. + arma::mat expectedOutput = arma::ones(3, 1); + expectedOutput *= -4; + expectedOutput = arma::exp(expectedOutput); + expectedOutput = arma::ones(3, 1) / (arma::ones(3, 1) + expectedOutput); + expectedOutput = (arma::ones(3, 1) - expectedOutput) % expectedOutput; + + // For the first input the output should be equal to the output of + // gate z_t as the previous output fed to the cell is all zeros. + REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); + + expectedOutput = output; + + gru.Forward(input, output); + + double s = arma::as_scalar(arma::sum(expectedOutput)); + + // Compute the value of z_t gate for the second input. + arma::mat z_t = arma::ones(3, 1); + z_t *= -(s + 4); + z_t = arma::exp(z_t); + z_t = arma::ones(3, 1) / (arma::ones(3, 1) + z_t); + + // Compute the value of o_t gate for the second input. + arma::mat o_t = arma::ones(3, 1); + o_t *= -(arma::as_scalar(arma::sum(expectedOutput % z_t)) + 4); + o_t = arma::exp(o_t); + o_t = arma::ones(3, 1) / (arma::ones(3, 1) + o_t); + + // Expected output for the second input. + expectedOutput = z_t % expectedOutput + (arma::ones(3, 1) - z_t) % o_t; + + REQUIRE(arma::as_scalar(arma::trans(output) * expectedOutput) <= 1e-2); + + LayerTypes<> layer(gruAlloc); + boost::apply_visitor(DeleteVisitor(), layer); +} /** * Simple concat module test. - * + */ TEST_CASE("SimpleConcatLayerTest", "[ANNLayerTest]") { arma::mat output, input, delta, error; - Linear* moduleA = new Linear(10, 10); + Linear<>* moduleA = new Linear<>(10, 10); moduleA->Parameters().randu(); moduleA->Reset(); - Linear* moduleB = new Linear(10, 10); + Linear<>* moduleB = new Linear<>(10, 10); moduleB->Parameters().randu(); moduleB->Reset(); - Concat module; + Concat<> module; module.Add(moduleA); module.Add(moduleB); @@ -2334,11 +1688,10 @@ TEST_CASE("SimpleConcatLayerTest", "[ANNLayerTest]") module.Backward(input, error, delta); REQUIRE(arma::accu(delta) == 0); } -*/ /** * Test to check Concat layer along different axes. - * + */ TEST_CASE("ConcatAlongAxisTest", "[ANNLayerTest]") { arma::mat output, input, error, outputA, outputB; @@ -2354,9 +1707,9 @@ TEST_CASE("ConcatAlongAxisTest", "[ANNLayerTest]") input = arma::ones(inputWidth * inputHeight * inputChannel, batch); - Convolution* moduleA = new Convolution(inputChannel, outputChannel, + Convolution<>* moduleA = new Convolution<>(inputChannel, outputChannel, kW, kH, 1, 1, 0, 0, inputWidth, inputHeight); - Convolution* moduleB = new Convolution(inputChannel, outputChannel, + Convolution<>* moduleB = new Convolution<>(inputChannel, outputChannel, kW, kH, 1, 1, 0, 0, inputWidth, inputHeight); moduleA->Reset(); @@ -2407,7 +1760,7 @@ TEST_CASE("ConcatAlongAxisTest", "[ANNLayerTest]") // Compute output of Concat<> layer. arma::Row inputSize{outputWidth, outputHeight, outputChannel}; - Concat module(inputSize, axis, true); + Concat<> module(inputSize, axis, true); module.Add(moduleA); module.Add(moduleB); module.Forward(input, output); @@ -2419,116 +1772,45 @@ TEST_CASE("ConcatAlongAxisTest", "[ANNLayerTest]") } delete moduleA; delete moduleB; -}*/ +} /** * Test that the function that can access the axis parameter of the * Concat layer works. - * + */ TEST_CASE("ConcatLayerParametersTest", "[ANNLayerTest]") { // Parameter order : inputSize{width, height, channels}, axis, model, run. arma::Row inputSize{128, 128, 3}; - Concat layer(inputSize, 2, false, true); + Concat<> layer(inputSize, 2, false, true); // Make sure we can get the parameters successfully. REQUIRE(layer.ConcatAxis() == 2); } -*/ /** * Concat layer numerical gradient test. */ -// TEST_CASE("GradientConcatLayerTest", "[ANNLayerTest]") -// { -// // Concat function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(); -// model->Add(10, 10); - -// concat = new Concat(true); -// concat->Add(10, 2); -// model->Add(concat); - -// model->Add(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// Concat* concat; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } - -/** - * Simple concatenate module test. - * -TEST_CASE("SimpleConcatenateLayerTest", "[ANNLayerTest]") +TEST_CASE("GradientConcatLayerTest", "[ANNLayerTest]") { - arma::mat input = arma::ones(5, 1); - arma::mat output, delta; - - Concatenate module; - module.Concat() = arma::ones(5, 1) * 0.5; - - // Test the Forward function. - module.Forward(input, output); - - REQUIRE(arma::accu(output) == 7.5); - - // Test the Backward function. - module.Backward(input, output, delta); - REQUIRE(arma::accu(delta) == 5); -} -*/ - -/** - * Concatenate layer numerical gradient test. - * -TEST_CASE("GradientConcatenateLayerTest", "[ANNLayerTest]") -{ - // Concatenate function gradient instantiation. + // Concat function gradient instantiation. struct GradientFunction { GradientFunction() : input(arma::randu(10, 1)), target(arma::mat("0")) { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 5); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 10); - arma::mat concat = arma::ones(5, 1); - // concatenate = new Concatenate(); - // concatenate->Concat() = concat; - // model->Add(concatenate); - model->Add(concat); + concat = new Concat<>(true); + concat->Add >(10, 2); + model->Add(concat); - model->Add(10, 5); - model->Add(); + model->Add >(); } ~GradientFunction() @@ -2545,18 +1827,87 @@ TEST_CASE("GradientConcatenateLayerTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; - Concatenate* concatenate; + FFN, NguyenWidrowInitialization>* model; + Concat<>* concat; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} + +/** + * Simple concatenate module test. + */ +TEST_CASE("SimpleConcatenateLayerTest", "[ANNLayerTest]") +{ + arma::mat input = arma::ones(5, 1); + arma::mat output, delta; + + Concatenate<> module; + module.Concat() = arma::ones(5, 1) * 0.5; + + // Test the Forward function. + module.Forward(input, output); + + REQUIRE(arma::accu(output) == 7.5); + + // Test the Backward function. + module.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 5); +} + +/** + * Concatenate layer numerical gradient test. + */ +TEST_CASE("GradientConcatenateLayerTest", "[ANNLayerTest]") +{ + // Concatenate function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 5); + + arma::mat concat = arma::ones(5, 1); + concatenate = new Concatenate<>(); + concatenate->Concat() = concat; + model->Add(concatenate); + + model->Add >(10, 5); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + Concatenate<>* concatenate; arma::mat input, target; } function; REQUIRE(CheckGradient(function) <= 1e-4); } -*/ /** * Simple lookup module test. - * + */ TEST_CASE("SimpleLookupLayerTest", "[ANNLayerTest]") { const size_t vocabSize = 10; @@ -2566,7 +1917,7 @@ TEST_CASE("SimpleLookupLayerTest", "[ANNLayerTest]") arma::mat output, input, gy, g, gradient; - Lookup module(vocabSize, embeddingSize); + Lookup<> module(vocabSize, embeddingSize); module.Parameters().randu(); // Test the Forward function. @@ -2593,11 +1944,10 @@ TEST_CASE("SimpleLookupLayerTest", "[ANNLayerTest]") REQUIRE(std::fabs(arma::accu(error) - arma::accu(gradient)) <= 1e-07); } -*/ /** * Lookup layer numerical gradient test. - * + */ TEST_CASE("GradientLookupLayerTest", "[ANNLayerTest]") { // Lookup function gradient instantiation. @@ -2618,10 +1968,11 @@ TEST_CASE("GradientLookupLayerTest", "[ANNLayerTest]") } model = new FFN, GlorotInitialization>(BCELoss<>(1e-10, false)); - model->ResetData(input, target); - model->Add(vocabSize, embeddingSize); - model->Add(embeddingSize * seqLength, vocabSize); - model->Add(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(vocabSize, embeddingSize); + model->Add >(embeddingSize * seqLength, vocabSize); + model->Add >(); } ~GradientFunction() @@ -2649,22 +2000,20 @@ TEST_CASE("GradientLookupLayerTest", "[ANNLayerTest]") REQUIRE(CheckGradient(function) <= 1e-6); } -*/ /** * Test that the functions that can access the parameters of the * Lookup layer work. - * + */ TEST_CASE("LookupLayerParametersTest", "[ANNLayerTest]") { // Parameter order : vocabSize, embedingSize. - Lookup layer(100, 8); + Lookup<> layer(100, 8); // Make sure we can get the parameters successfully. REQUIRE(layer.VocabSize() == 100); REQUIRE(layer.EmbeddingSize() == 8); } -*/ /** * Simple LogSoftMax module test. @@ -2672,7 +2021,7 @@ TEST_CASE("LookupLayerParametersTest", "[ANNLayerTest]") TEST_CASE("SimpleLogSoftmaxLayerTest", "[ANNLayerTest]") { arma::mat output, input, error, delta; - LogSoftMax module; + LogSoftMax<> module; // Test the Forward function. input = arma::mat("0.5; 0.5"); @@ -2691,11 +2040,11 @@ TEST_CASE("SimpleLogSoftmaxLayerTest", "[ANNLayerTest]") /** * Simple Softmax module test. - * + */ TEST_CASE("SimpleSoftmaxLayerTest", "[ANNLayerTest]") { arma::mat input, output, gy, g; - Softmax module; + Softmax<> module; // Test the forward function. input = arma::mat("1.7; 3.6"); @@ -2710,11 +2059,10 @@ TEST_CASE("SimpleSoftmaxLayerTest", "[ANNLayerTest]") REQUIRE(arma::accu(arma::abs(arma::mat("0.11318; -0.11318") - g)) == Approx(0.0).margin(1e-04)); } -*/ /** * Softmax layer numerical gradient test. - * + */ TEST_CASE("GradientSoftmaxTest", "[ANNLayerTest]") { // Softmax function gradient instantiation. @@ -2724,12 +2072,13 @@ TEST_CASE("GradientSoftmaxTest", "[ANNLayerTest]") input(arma::randu(10, 1)), target(arma::mat("1; 0")) { - model = new FFN; - model->ResetData(input, target); - model->Add(10, 10); - model->Add(); - model->Add(10, 2); - model->Add(); + model = new FFN, RandomInitialization>; + model->Predictors() = input; + model->Responses() = target; + model->Add >(10, 10); + model->Add >(); + model->Add >(10, 2); + model->Add >(); } ~GradientFunction() @@ -2746,17 +2095,16 @@ TEST_CASE("GradientSoftmaxTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN >* model; arma::mat input, target; } function; REQUIRE(CheckGradient(function) <= 1e-4); } -*/ -/** +/* * Simple test for the NearestInterpolation layer - * + */ TEST_CASE("SimpleNearestInterpolationLayerTest", "[ANNLayerTest]") { // Tested output against torch.nn.Upsample(mode="nearest"). @@ -2816,11 +2164,10 @@ TEST_CASE("SimpleNearestInterpolationLayerTest", "[ANNLayerTest]") REQUIRE(arma::accu(unzoomedOutput1) - 1317.00 == Approx(0.0).margin(1e-05)); } -*/ /* * Simple test for the BilinearInterpolation layer - * + */ TEST_CASE("SimpleBilinearInterpolationLayerTest", "[ANNLayerTest]") { // Tested output against tensorflow.image.resize_bilinear() @@ -2834,7 +2181,7 @@ TEST_CASE("SimpleBilinearInterpolationLayerTest", "[ANNLayerTest]") input[0] = 1.0; input[1] = input[2] = 2.0; input[3] = 3.0; - BilinearInterpolation layer(inRowSize, inColSize, outRowSize, outColSize, + BilinearInterpolation<> layer(inRowSize, inColSize, outRowSize, outColSize, depth); expectedOutput = arma::mat("1.0000 1.4000 1.8000 2.0000 2.0000 \ 1.4000 1.8000 2.2000 2.4000 2.4000 \ @@ -2851,17 +2198,16 @@ TEST_CASE("SimpleBilinearInterpolationLayerTest", "[ANNLayerTest]") CheckMatrices(unzoomedOutput - expectedOutput, arma::zeros(input.n_rows), 1e-12); } -*/ /** * Test that the functions that can modify and access the parameters of the * Bilinear Interpolation layer work. - * + */ TEST_CASE("BilinearInterpolationLayerParametersTest", "[ANNLayerTest]") { // Parameter order : inRowSize, inColSize, outRowSize, outColSize, depth. - BilinearInterpolation layer1(1, 2, 3, 4, 5); - BilinearInterpolation layer2(2, 3, 4, 5, 6); + BilinearInterpolation<> layer1(1, 2, 3, 4, 5); + BilinearInterpolation<> layer2(2, 3, 4, 5, 6); // Make sure we can get the parameters successfully. REQUIRE(layer1.InRowSize() == 1); @@ -2884,11 +2230,10 @@ TEST_CASE("BilinearInterpolationLayerParametersTest", "[ANNLayerTest]") REQUIRE(layer1.OutColSize() == layer2.OutColSize()); REQUIRE(layer1.InDepth() == layer2.InDepth()); } -*/ /* * Simple test for the BicubicInterpolation layer. - * + */ TEST_CASE("SimpleBicubicInterpolationLayerTest", "[ANNLayerTest]") { // Tested output against torch.nn.Upsample(mode="bicubic"). @@ -2983,171 +2328,171 @@ TEST_CASE("SimpleBicubicInterpolationLayerTest", "[ANNLayerTest]") layer1.Backward(output1, output1, unzoomedOutput1); CheckMatrices(unzoomedOutput1, expectedUnzoomed, 1e-6); } -*/ -// /** -// * Tests the BatchNorm Layer, compares the layers parameters with -// * the values from another implementation. -// * Link to the implementation - http://cthorey.github.io./backpropagation/ -// */ -// TEST_CASE("BatchNormTest", "[ANNLayerTest]") -// { -// arma::mat input, output; -// input << 5.1 << 3.5 << 1.4 << arma::endr -// << 4.9 << 3.0 << 1.4 << arma::endr -// << 4.7 << 3.2 << 1.3 << arma::endr; +/** + * Tests the BatchNorm Layer, compares the layers parameters with + * the values from another implementation. + * Link to the implementation - http://cthorey.github.io./backpropagation/ + */ +TEST_CASE("BatchNormTest", "[ANNLayerTest]") +{ + arma::mat input, output; + input = { { 5.1, 3.5, 1.4 }, + { 4.9, 3.0, 1.4 }, + { 4.7, 3.2, 1.3 } }; -// // BatchNorm layer with average parameter set to true. -// BatchNorm<> model(input.n_rows); -// model.Reset(); + // BatchNorm layer with average parameter set to true. + BatchNorm<> model(input.n_rows); + model.Reset(); -// // BatchNorm layer with average parameter set to false. -// BatchNorm<> model2(input.n_rows, 1e-5, false); -// model2.Reset(); + // BatchNorm layer with average parameter set to false. + BatchNorm<> model2(input.n_rows, 1e-5, false); + model2.Reset(); -// // Non-Deteministic Forward Pass Test. -// model.Deterministic() = false; -// model.Forward(input, output); + // Non-Deteministic Forward Pass Test. + model.Deterministic() = false; + model.Forward(input, output); -// // Value calculates using torch.nn.BatchNorm2d(momentum = None). -// arma::mat result; -// result = { { 1.1658, 0.1100, -1.2758 }, -// { 1.2579, -0.0699, -1.1880}, -// { 1.1737, 0.0958, -1.2695 } }; + // Value calculates using torch.nn.BatchNorm2d(momentum = None). + arma::mat result; + result = { { 1.1658, 0.1100, -1.2758 }, + { 1.2579, -0.0699, -1.1880}, + { 1.1737, 0.0958, -1.2695 } }; -// CheckMatrices(output, result, 1e-1); + CheckMatrices(output, result, 1e-1); -// model2.Forward(input, output); -// CheckMatrices(output, result, 1e-1); -// result.clear(); - -// // Values calculated using torch.nn.BatchNorm2d(momentum = None). -// output = model.TrainingMean(); -// result = arma::mat({ 3.33333333, 3.1, 3.06666666 }).t(); - -// CheckMatrices(output, result, 1e-1); - -// // Values calculated using torch.nn.BatchNorm2d(). -// output = model2.TrainingMean(); -// result = arma::mat({ 0.3333, 0.3100, 0.3067 }).t(); - -// CheckMatrices(output, result, 1e-1); -// result.clear(); + model2.Forward(input, output); + CheckMatrices(output, result, 1e-1); + result.clear(); // Values calculated using torch.nn.BatchNorm2d(momentum = None). -// output = model.TrainingVariance(); -// result = arma::mat({ 3.4433, 3.0700, 2.9033 }).t(); + output = model.TrainingMean(); + result = arma::mat({ 3.33333333, 3.1, 3.06666666 }).t(); -// CheckMatrices(output, result, 1e-1); -// result.clear(); + CheckMatrices(output, result, 1e-1); // Values calculated using torch.nn.BatchNorm2d(). -// output = model2.TrainingVariance(); -// result = arma::mat({ 1.2443, 1.2070, 1.1903 }).t(); + output = model2.TrainingMean(); + result = arma::mat({ 0.3333, 0.3100, 0.3067 }).t(); -// CheckMatrices(output, result, 1e-1); -// result.clear(); - -// // Deterministic Forward Pass test. -// model.Deterministic() = true; -// model.Forward(input, output); + CheckMatrices(output, result, 1e-1); + result.clear(); // Values calculated using torch.nn.BatchNorm2d(momentum = None). -// result = { { 0.9521, 0.0898, -1.0419 }, -// { 1.0273, -0.0571, -0.9702 }, -// { 0.9586, 0.0783, -1.0368 } }; + output = model.TrainingVariance(); + result = arma::mat({ 3.4433, 3.0700, 2.9033 }).t(); -// CheckMatrices(output, result, 1e-1); + CheckMatrices(output, result, 1e-1); + result.clear(); -// // Values calculated using torch.nn.BatchNorm2d(). -// model2.Deterministic() = true; -// model2.Forward(input, output); + // Values calculated using torch.nn.BatchNorm2d(). + output = model2.TrainingVariance(); + result = arma::mat({ 1.2443, 1.2070, 1.1903 }).t(); -// result = { { 4.2731, 2.8388, 0.9562 }, -// { 4.1779, 2.4485, 0.9921 }, -// { 4.0268, 2.6519, 0.9105 } }; -// -// CheckMatrices(output, result, 1e-1); -// } + CheckMatrices(output, result, 1e-1); + result.clear(); -// /** -// * BatchNorm layer numerical gradient test. -// */ -// TEST_CASE("GradientBatchNormTest", "[ANNLayerTest]") -// { -// bool pass = false; -// for (size_t trial = 0; trial < 10; trial++) -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(32, 2048)), -// target(arma::zeros(1, 2048)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(32, 4); -// model->Add >(4); -// model->Add>(4, 2); -// model->Add >(); -// } + // Deterministic Forward Pass test. + model.Deterministic() = true; + model.Forward(input, output); -// ~GradientFunction() -// { -// delete model; -// } + // Values calculated using torch.nn.BatchNorm2d(momentum = None). + result = { { 0.9521, 0.0898, -1.0419 }, + { 1.0273, -0.0571, -0.9702 }, + { 0.9586, 0.0783, -1.0368 } }; -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 2048, false); -// model->Gradient(model->Parameters(), 0, gradient, 2048); -// return error; -// } + CheckMatrices(output, result, 1e-1); -// arma::mat& Parameters() { return model->Parameters(); } + // Values calculated using torch.nn.BatchNorm2d(). + model2.Deterministic() = true; + model2.Forward(input, output); -// FFN* model; -// arma::mat input, target; -// } function; + result = { { 4.2731, 2.8388, 0.9562 }, + { 4.1779, 2.4485, 0.9921 }, + { 4.0268, 2.6519, 0.9105 } }; -// double gradient = CheckGradient(function); -// if (gradient < 2e-1) -// { -// pass = true; -// break; -// } -// } + CheckMatrices(output, result, 1e-1); +} -// REQUIRE(pass); -// } +/** + * BatchNorm layer numerical gradient test. + */ +TEST_CASE("GradientBatchNormTest", "[ANNLayerTest]") +{ + bool pass = false; + for (size_t trial = 0; trial < 10; trial++) + { + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randn(32, 2048)), + target(arma::zeros(1, 2048)) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(32, 4); + model->Add >(4); + model->Add>(4, 2); + model->Add >(); + } -// /** -// * Test that the functions that can access the parameters of the -// * Batch Norm layer work. -// */ -// TEST_CASE("BatchNormLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order : size, eps. -// BatchNorm<> layer(7, 1e-3); + ~GradientFunction() + { + delete model; + } -// // Make sure we can get the parameters successfully. -// REQUIRE(layer.InputSize() == 7); -// REQUIRE(layer.Epsilon() == 1e-3); + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 2048, false); + model->Gradient(model->Parameters(), 0, gradient, 2048); + return error; + } -// arma::mat runningMean(7, 1, arma::fill::randn); -// arma::mat runningVariance(7, 1, arma::fill::randn); + arma::mat& Parameters() { return model->Parameters(); } -// layer.TrainingVariance() = runningVariance; -// layer.TrainingMean() = runningMean; -// CheckMatrices(layer.TrainingVariance(), runningVariance); -// CheckMatrices(layer.TrainingMean(), runningMean); -// } + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + double gradient = CheckGradient(function); + if (gradient < 2e-1) + { + pass = true; + break; + } + } + + REQUIRE(pass); +} + +/** + * Test that the functions that can access the parameters of the + * Batch Norm layer work. + */ +TEST_CASE("BatchNormLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : size, eps. + BatchNorm<> layer(7, 1e-3); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.InputSize() == 7); + REQUIRE(layer.Epsilon() == 1e-3); + + arma::mat runningMean(7, 1, arma::fill::randn); + arma::mat runningVariance(7, 1, arma::fill::randn); + + layer.TrainingVariance() = runningVariance; + layer.TrainingMean() = runningMean; + CheckMatrices(layer.TrainingVariance(), runningVariance); + CheckMatrices(layer.TrainingMean(), runningMean); +} /** * VirtualBatchNorm layer numerical gradient test. - * + */ TEST_CASE("GradientVirtualBatchNormTest", "[ANNLayerTest]") { // Add function gradient instantiation. @@ -3157,15 +2502,16 @@ TEST_CASE("GradientVirtualBatchNormTest", "[ANNLayerTest]") input(arma::randn(5, 256)), target(arma::zeros(1, 256)) { - arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); + arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 16); - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(5, 5); - model->Add(referenceBatch, 5); - model->Add(5, 2); - model->Add(); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(5, 5); + model->Add >(referenceBatch, 5); + model->Add >(5, 2); + model->Add >(); } ~GradientFunction() @@ -3175,87 +2521,86 @@ TEST_CASE("GradientVirtualBatchNormTest", "[ANNLayerTest]") double Gradient(arma::mat& gradient) const { - double error = model->Evaluate(model->Parameters(), 0, 16, false); - model->Gradient(model->Parameters(), 0, gradient, 16); + double error = model->Evaluate(model->Parameters(), 0, 256, false); + model->Gradient(model->Parameters(), 0, gradient, 256); return error; } arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, NguyenWidrowInitialization>* model; arma::mat input, target; } function; REQUIRE(CheckGradient(function) <= 1e-4); } -*/ /** * Test that the functions that can modify and access the parameters of the * Virtual Batch Norm layer work. - * + */ TEST_CASE("VirtualBatchNormLayerParametersTest", "[ANNLayerTest]") { - arma::mat input = arma::randn(5, 16); - arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 4); + arma::mat input = arma::randn(5, 256); + arma::mat referenceBatch = arma::mat(input.memptr(), input.n_rows, 16); // Parameter order : referenceBatch, size, eps. - VirtualBatchNorm layer(referenceBatch, 5, 1e-3); + VirtualBatchNorm<> layer(referenceBatch, 5, 1e-3); // Make sure we can get the parameters successfully. REQUIRE(layer.InSize() == 5); REQUIRE(layer.Epsilon() == 1e-3); } -*/ -// /** -// * MiniBatchDiscrimination layer numerical gradient test. -// */ -// TEST_CASE("MiniBatchDiscriminationTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(5, 4)), -// target(arma::zeros(1, 4)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(5, 5); -// model->Add >(5, 10, 16); -// model->Add >(10, 2); -// model->Add >(); -// } +/** + * MiniBatchDiscrimination layer numerical gradient test. + */ +TEST_CASE("MiniBatchDiscriminationTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randn(5, 4)), + target(arma::zeros(1, 4)) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(5, 5); + model->Add >(5, 10, 16); + model->Add >(10, 2); + model->Add >(); + } -// ~GradientFunction() -// { -// delete model; -// } + ~GradientFunction() + { + delete model; + } -// double Gradient(arma::mat& gradient) const -// { -// return model->EvaluateWithGradient(model->Parameters(), 0, gradient, 4); -// } + double Gradient(arma::mat& gradient) const + { + return model->EvaluateWithGradient(model->Parameters(), 0, gradient, 4); + } -// arma::mat& Parameters() { return model->Parameters(); } + arma::mat& Parameters() { return model->Parameters(); } -// FFN* model; -// arma::mat input, target; -// } function; + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; -// REQUIRE(CheckGradient(function) <= 1e-4); -// } + REQUIRE(CheckGradient(function) <= 1e-4); +} /** * Simple Transposed Convolution layer test. - * + */ TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") { arma::mat output, input, delta; - TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6); + TransposedConvolution<> module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6); // Test the forward function. input = arma::linspace(0, 15, 16); module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); @@ -3271,7 +2616,7 @@ TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") // Value calculated using tensorflow.nn.conv2d() REQUIRE(arma::accu(delta) == 720.0); - TransposedConvolution module2(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 6, 6); + TransposedConvolution<> module2(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 6, 6); // Test the forward function. input = arma::linspace(0, 24, 25); module2.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); @@ -3291,7 +2636,7 @@ TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") // Value calculated using torch.nn.functional.conv2d() REQUIRE(arma::accu(delta) == 6504.0); - TransposedConvolution module3(1, 1, 3, 3, 1, 1, 1, 1, 5, 5, 5, 5); + TransposedConvolution<> module3(1, 1, 3, 3, 1, 1, 1, 1, 5, 5, 5, 5); // Test the forward function. input = arma::linspace(0, 24, 25); module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); @@ -3309,7 +2654,7 @@ TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") // Value calculated using torch.nn.functional.conv2d() REQUIRE(arma::accu(delta) == 19154.0); - TransposedConvolution module4(1, 1, 3, 3, 1, 1, 0, 0, 5, 5, 7, 7); + TransposedConvolution<> module4(1, 1, 3, 3, 1, 1, 0, 0, 5, 5, 7, 7); // Test the forward function. input = arma::linspace(0, 24, 25); module4.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); @@ -3327,7 +2672,7 @@ TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") // Value calculated using torch.nn.functional.conv2d() REQUIRE(arma::accu(delta) == 86208.0); - TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 5, 5); + TransposedConvolution<> module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 5, 5); // Test the forward function. input = arma::linspace(0, 3, 4); module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); @@ -3345,7 +2690,7 @@ TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") // Value calculated using torch.nn.functional.conv2d() REQUIRE(arma::accu(delta) == 960.0); - TransposedConvolution module6(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 5, 5); + TransposedConvolution<> module6(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 5, 5); // Test the forward function. input = arma::linspace(0, 8, 9); module6.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); @@ -3363,7 +2708,7 @@ TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") // Value calculated using torch.nn.functional.conv2d() REQUIRE(arma::accu(delta) == 4444.0); - TransposedConvolution module7(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 6, 6); + TransposedConvolution<> module7(1, 1, 3, 3, 2, 2, 1, 1, 3, 3, 6, 6); // Test the forward function. input = arma::linspace(0, 8, 9); module7.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); @@ -3380,11 +2725,10 @@ TEST_CASE("SimpleTransposedConvolutionLayerTest", "[ANNLayerTest]") // Value calculated using torch.nn.functional.conv2d() REQUIRE(arma::accu(delta) == 7732.0); } -*/ /** * Transposed Convolution layer numerical gradient test. - * + */ TEST_CASE("GradientTransposedConvolutionLayerTest", "[ANNLayerTest]") { // Add function gradient instantiation. @@ -3398,10 +2742,12 @@ TEST_CASE("GradientTransposedConvolutionLayerTest", "[ANNLayerTest]") input(arma::linspace(0, 35, 36)), target(arma::mat("0")) { - model = new FFN(); - model->ResetData(input, target); - model->Add(1, 1, 3, 3, 2, 2, 1, 1, 6, 6, 12, 12); - model->Add(); + model = new FFN, RandomInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add > + (1, 1, 3, 3, 2, 2, 1, 1, 6, 6, 12, 12); + model->Add >(); } ~GradientFunction() @@ -3418,7 +2764,7 @@ TEST_CASE("GradientTransposedConvolutionLayerTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, RandomInitialization>* model; arma::mat input, target; } function; @@ -3430,11 +2776,10 @@ TEST_CASE("GradientTransposedConvolutionLayerTest", "[ANNLayerTest]") } REQUIRE(pass == true); } -*/ /** * Simple MultiplyMerge module test. - * + */ TEST_CASE("SimpleMultiplyMergeLayerTest", "[ANNLayerTest]") { arma::mat output, input, delta; @@ -3442,14 +2787,14 @@ TEST_CASE("SimpleMultiplyMergeLayerTest", "[ANNLayerTest]") for (size_t i = 0; i < 5; ++i) { - MultiplyMerge module(false, false); + MultiplyMerge<> module(false, false); const size_t numMergeModules = math::RandInt(2, 10); for (size_t m = 0; m < numMergeModules; ++m) { - IdentityLayer* identityLayer = new IdentityLayer(); - identityLayer->Forward(input, identityLayer->OutputParameter()); + IdentityLayer<> identityLayer; + identityLayer.Forward(input, identityLayer.OutputParameter()); - module.Add(identityLayer); + module.Add >(identityLayer); } // Test the Forward function. @@ -3461,324 +2806,325 @@ TEST_CASE("SimpleMultiplyMergeLayerTest", "[ANNLayerTest]") REQUIRE(arma::accu(output) == arma::accu(delta)); } } -*/ /** * Check whether copying and moving network with MultiplyMerge is working or * not. */ -// TEST_CASE("CheckCopyMoveMultiplyMergeTest", "[ANNLayerTest]") -// { -// arma::mat input(10, 1); -// input.randu(); -// -// arma::mat output1; -// arma::mat output2; -// arma::mat output3; -// arma::mat output4; -// -// const size_t numMergeModules = math::RandInt(2, 10); -// -// MultiplyMerge<> *module1 = new MultiplyMerge<>(true, false); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); -// -// module1->Add >(identityLayer); -// } -// -// module1->Forward(input, output1); -// -// MultiplyMerge<> module2 = *module1; -// delete module1; -// -// module2.Forward(input, output2); -// CheckMatrices(output1, output2); -// -// MultiplyMerge<> *module3 = new MultiplyMerge<>(true, false); -// for (size_t m = 0; m < numMergeModules; ++m) -// { -// IdentityLayer<> identityLayer; -// identityLayer.Forward(input, identityLayer.OutputParameter()); -// -// module3->Add >(identityLayer); -// } -// module3->Forward(input, output3); -// -// MultiplyMerge<> module4(std::move(*module3)); -// delete module3; -// -// module4.Forward(input, output4); -// CheckMatrices(output3, output4); -// } +TEST_CASE("CheckCopyMoveMultiplyMergeTest", "[ANNLayerTest]") +{ + arma::mat input(10, 1); + input.randu(); -// /** -// * Simple Atrous Convolution layer test. -// */ -// TEST_CASE("SimpleAtrousConvolutionLayerTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; + arma::mat output1; + arma::mat output2; + arma::mat output3; + arma::mat output4; -// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, 0, 0, 7, 7, 2, 2); -// // Test the Forward function. -// input = arma::linspace(0, 48, 49); -// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module1.Parameters()(0) = 1.0; -// module1.Parameters()(8) = 2.0; -// module1.Reset(); -// module1.Forward(input, output); -// // Value calculated using tensorflow.nn.atrous_conv2d() -// REQUIRE(arma::accu(output) == 792.0); + const size_t numMergeModules = math::RandInt(2, 10); -// // Test the Backward function. -// module1.Backward(input, output, delta); -// REQUIRE(arma::accu(delta) == 2376); + MultiplyMerge<> *module1 = new MultiplyMerge<>(true, false); + for (size_t m = 0; m < numMergeModules; ++m) + { + IdentityLayer<> identityLayer; + identityLayer.Forward(input, identityLayer.OutputParameter()); -// AtrousConvolution<> module2(1, 1, 3, 3, 2, 2, 0, 0, 7, 7, 2, 2); -// // Test the forward function. -// input = arma::linspace(0, 48, 49); -// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module2.Parameters()(0) = 1.0; -// module2.Parameters()(3) = 1.0; -// module2.Parameters()(6) = 1.0; -// module2.Reset(); -// module2.Forward(input, output); -// // Value calculated using tensorflow.nn.conv2d() -// REQUIRE(arma::accu(output) == 264.0); + module1->Add >(identityLayer); + } -// // Test the backward function. -// module2.Backward(input, output, delta); -// REQUIRE(arma::accu(delta) == 792.0); -// } + module1->Forward(input, output1); -// /** -// * Atrous Convolution layer numerical gradient test. -// */ -// TEST_CASE("GradientAtrousConvolutionLayerTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::linspace(0, 35, 36)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(1, 1, 3, 3, 1, 1, 0, 0, 6, 6, 2, 2); -// model->Add >(); -// } + MultiplyMerge<> module2 = *module1; + delete module1; -// ~GradientFunction() -// { -// delete model; -// } + module2.Forward(input, output2); + CheckMatrices(output1, output2); -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } + MultiplyMerge<> *module3 = new MultiplyMerge<>(true, false); + for (size_t m = 0; m < numMergeModules; ++m) + { + IdentityLayer<> identityLayer; + identityLayer.Forward(input, identityLayer.OutputParameter()); -// arma::mat& Parameters() { return model->Parameters(); } + module3->Add >(identityLayer); + } + module3->Forward(input, output3); -// FFN* model; -// arma::mat input, target; -// } function; + MultiplyMerge<> module4(std::move(*module3)); + delete module3; -// // TODO: this tolerance seems far higher than necessary. The implementation -// // should be checked. -// REQUIRE(CheckGradient(function) <= 0.2); -// } + module4.Forward(input, output4); + CheckMatrices(output3, output4); +} -// /** -// * Test the functions to access and modify the parameters of the -// * AtrousConvolution layer. -// */ -// TEST_CASE("AtrousConvolutionLayerParametersTest", "[ANNLayerTest]") -// { -// // Parameter order for the constructor: inSize, outSize, kW, kH, dW, dH, padW, -// // padH, inputWidth, inputHeight, dilationW, dilationH, paddingType ("none"). -// AtrousConvolution<> layer1(1, 2, 3, 4, 5, 6, std::make_tuple(7, 8), -// std::make_tuple(9, 10), 11, 12, 13, 14); -// AtrousConvolution<> layer2(2, 3, 4, 5, 6, 7, std::make_tuple(8, 9), -// std::make_tuple(10, 11), 12, 13, 14, 15); +/** + * Simple Atrous Convolution layer test. + */ +TEST_CASE("SimpleAtrousConvolutionLayerTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; -// // Make sure we can get the parameters successfully. -// REQUIRE(layer1.InputWidth() == 11); -// REQUIRE(layer1.InputHeight() == 12); -// REQUIRE(layer1.KernelWidth() == 3); -// REQUIRE(layer1.KernelHeight() == 4); -// REQUIRE(layer1.StrideWidth() == 5); -// REQUIRE(layer1.StrideHeight() == 6); -// REQUIRE(layer1.Padding().PadHTop() == 9); -// REQUIRE(layer1.Padding().PadHBottom() == 10); -// REQUIRE(layer1.Padding().PadWLeft() == 7); -// REQUIRE(layer1.Padding().PadWRight() == 8); -// REQUIRE(layer1.DilationWidth() == 13); -// REQUIRE(layer1.DilationHeight() == 14); + AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, 0, 0, 7, 7, 2, 2); + // Test the Forward function. + input = arma::linspace(0, 48, 49); + module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module1.Parameters()(0) = 1.0; + module1.Parameters()(8) = 2.0; + module1.Reset(); + module1.Forward(input, output); + // Value calculated using tensorflow.nn.atrous_conv2d() + REQUIRE(arma::accu(output) == 792.0); -// // Now modify the parameters to match the second layer. -// layer1.InputWidth() = 12; -// layer1.InputHeight() = 13; -// layer1.KernelWidth() = 4; -// layer1.KernelHeight() = 5; -// layer1.StrideWidth() = 6; -// layer1.StrideHeight() = 7; -// layer1.Padding().PadHTop() = 10; -// layer1.Padding().PadHBottom() = 11; -// layer1.Padding().PadWLeft() = 8; -// layer1.Padding().PadWRight() = 9; -// layer1.DilationWidth() = 14; -// layer1.DilationHeight() = 15; + // Test the Backward function. + module1.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 2376); -// // Now ensure all results are the same. -// REQUIRE(layer1.InputWidth() == layer2.InputWidth()); -// REQUIRE(layer1.InputHeight() == layer2.InputHeight()); -// REQUIRE(layer1.KernelWidth() == layer2.KernelWidth()); -// REQUIRE(layer1.KernelHeight() == layer2.KernelHeight()); -// REQUIRE(layer1.StrideWidth() == layer2.StrideWidth()); -// REQUIRE(layer1.StrideHeight() == layer2.StrideHeight()); -// REQUIRE(layer1.Padding().PadHTop() == layer2.Padding().PadHTop()); -// REQUIRE(layer1.Padding().PadHBottom() == -// layer2.Padding().PadHBottom()); -// REQUIRE(layer1.Padding().PadWLeft() == -// layer2.Padding().PadWLeft()); -// REQUIRE(layer1.Padding().PadWRight() == -// layer2.Padding().PadWRight()); -// REQUIRE(layer1.DilationWidth() == layer2.DilationWidth()); -// REQUIRE(layer1.DilationHeight() == layer2.DilationHeight()); -// } + AtrousConvolution<> module2(1, 1, 3, 3, 2, 2, 0, 0, 7, 7, 2, 2); + // Test the forward function. + input = arma::linspace(0, 48, 49); + module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module2.Parameters()(0) = 1.0; + module2.Parameters()(3) = 1.0; + module2.Parameters()(6) = 1.0; + module2.Reset(); + module2.Forward(input, output); + // Value calculated using tensorflow.nn.conv2d() + REQUIRE(arma::accu(output) == 264.0); -// /** -// * Test that the padding options are working correctly in Atrous Convolution -// * layer. -// */ -// TEST_CASE("AtrousConvolutionLayerPaddingTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta; + // Test the backward function. + module2.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == 792.0); +} -// // Check valid padding option. -// AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, -// std::tuple(1, 1), std::tuple(1, 1), 7, 7, -// 2, 2, "valid"); +/** + * Atrous Convolution layer numerical gradient test. + */ +TEST_CASE("GradientAtrousConvolutionLayerTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::linspace(0, 35, 36)), + target(arma::mat("0")) + { + model = new FFN, RandomInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(1, 1, 3, 3, 1, 1, 0, 0, 6, 6, 2, 2); + model->Add >(); + } -// // Test the Forward function. -// input = arma::linspace(0, 48, 49); -// module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module1.Reset(); -// module1.Forward(input, output); + ~GradientFunction() + { + delete model; + } -// REQUIRE(arma::accu(output) == 0); -// REQUIRE(output.n_rows == 9); -// REQUIRE(output.n_cols == 1); + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } -// // Test the Backward function. -// module1.Backward(input, output, delta); + arma::mat& Parameters() { return model->Parameters(); } -// // Check same padding option. -// AtrousConvolution<> module2(1, 1, 3, 3, 1, 1, -// std::tuple(0, 0), std::tuple(0, 0), 7, 7, -// 2, 2, "same"); + FFN, RandomInitialization>* model; + arma::mat input, target; + } function; -// // Test the forward function. -// input = arma::linspace(0, 48, 49); -// module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); -// module2.Reset(); -// module2.Forward(input, output); + // TODO: this tolerance seems far higher than necessary. The implementation + // should be checked. + REQUIRE(CheckGradient(function) <= 0.2); +} -// REQUIRE(arma::accu(output) == 0); -// REQUIRE(output.n_rows == 49); -// REQUIRE(output.n_cols == 1); +/** + * Test the functions to access and modify the parameters of the + * AtrousConvolution layer. + */ +TEST_CASE("AtrousConvolutionLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order for the constructor: inSize, outSize, kW, kH, dW, dH, padW, + // padH, inputWidth, inputHeight, dilationW, dilationH, paddingType ("none"). + AtrousConvolution<> layer1(1, 2, 3, 4, 5, 6, std::make_tuple(7, 8), + std::make_tuple(9, 10), 11, 12, 13, 14); + AtrousConvolution<> layer2(2, 3, 4, 5, 6, 7, std::make_tuple(8, 9), + std::make_tuple(10, 11), 12, 13, 14, 15); -// // Test the backward function. -// module2.Backward(input, output, delta); -// } + // Make sure we can get the parameters successfully. + REQUIRE(layer1.InputWidth() == 11); + REQUIRE(layer1.InputHeight() == 12); + REQUIRE(layer1.KernelWidth() == 3); + REQUIRE(layer1.KernelHeight() == 4); + REQUIRE(layer1.StrideWidth() == 5); + REQUIRE(layer1.StrideHeight() == 6); + REQUIRE(layer1.Padding().PadHTop() == 9); + REQUIRE(layer1.Padding().PadHBottom() == 10); + REQUIRE(layer1.Padding().PadWLeft() == 7); + REQUIRE(layer1.Padding().PadWRight() == 8); + REQUIRE(layer1.DilationWidth() == 13); + REQUIRE(layer1.DilationHeight() == 14); + + // Now modify the parameters to match the second layer. + layer1.InputWidth() = 12; + layer1.InputHeight() = 13; + layer1.KernelWidth() = 4; + layer1.KernelHeight() = 5; + layer1.StrideWidth() = 6; + layer1.StrideHeight() = 7; + layer1.Padding().PadHTop() = 10; + layer1.Padding().PadHBottom() = 11; + layer1.Padding().PadWLeft() = 8; + layer1.Padding().PadWRight() = 9; + layer1.DilationWidth() = 14; + layer1.DilationHeight() = 15; + + // Now ensure all results are the same. + REQUIRE(layer1.InputWidth() == layer2.InputWidth()); + REQUIRE(layer1.InputHeight() == layer2.InputHeight()); + REQUIRE(layer1.KernelWidth() == layer2.KernelWidth()); + REQUIRE(layer1.KernelHeight() == layer2.KernelHeight()); + REQUIRE(layer1.StrideWidth() == layer2.StrideWidth()); + REQUIRE(layer1.StrideHeight() == layer2.StrideHeight()); + REQUIRE(layer1.Padding().PadHTop() == layer2.Padding().PadHTop()); + REQUIRE(layer1.Padding().PadHBottom() == + layer2.Padding().PadHBottom()); + REQUIRE(layer1.Padding().PadWLeft() == + layer2.Padding().PadWLeft()); + REQUIRE(layer1.Padding().PadWRight() == + layer2.Padding().PadWRight()); + REQUIRE(layer1.DilationWidth() == layer2.DilationWidth()); + REQUIRE(layer1.DilationHeight() == layer2.DilationHeight()); +} + +/** + * Test that the padding options are working correctly in Atrous Convolution + * layer. + */ +TEST_CASE("AtrousConvolutionLayerPaddingTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta; + + // Check valid padding option. + AtrousConvolution<> module1(1, 1, 3, 3, 1, 1, + std::tuple(1, 1), std::tuple(1, 1), 7, 7, + 2, 2, "valid"); + + // Test the Forward function. + input = arma::linspace(0, 48, 49); + module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module1.Reset(); + module1.Forward(input, output); + + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == 9); + REQUIRE(output.n_cols == 1); + + // Test the Backward function. + module1.Backward(input, output, delta); + + // Check same padding option. + AtrousConvolution<> module2(1, 1, 3, 3, 1, 1, + std::tuple(0, 0), std::tuple(0, 0), 7, 7, + 2, 2, "same"); + + // Test the forward function. + input = arma::linspace(0, 48, 49); + module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module2.Reset(); + module2.Forward(input, output); + + REQUIRE(arma::accu(output) == 0); + REQUIRE(output.n_rows == 49); + REQUIRE(output.n_cols == 1); + + // Test the backward function. + module2.Backward(input, output, delta); +} /** * Tests the GroupNorm layer. */ -// TEST_CASE("GroupNormTest", "[ANNLayerTest]") -// { -// arma::mat input, output, backwardOutput; -// input = { -// { 2, 0, 1 }, -// { 3, 1, 2 }, -// { 5, 1, 3 }, -// { 7, 2, 4 }, -// { 11, 3, 5 }, -// { 13, 5, 6 }, -// { 17, 8, 7 }, -// { 19, 13, 8 } -// }; -// -// GroupNorm<> model(2, 4); -// model.Reset(); -// -// model.Forward(input, output); -// arma::mat result; -// result = { -// { -1.1717001972, -1.4142135482, -1.3416407811 }, -// { -0.6509445540, 0.0000000000 , -0.4472135937 }, -// { 0.3905667324 , 0.0000000000 , 0.4472135937 }, -// { 1.4320780188 , 1.4142135482 , 1.341640781 }, -// { -1.2649110634, -1.1283296293, -1.3416407811 }, -// { -0.6324555317, -0.5973509802, -0.4472135937 }, -// { 0.6324555317 , 0.1991169934 , 0.4472135937 }, -// { 1.2649110634 , 1.5265636161 , 1.3416407811 } -// }; -// -// CheckMatrices(output, result, 1e-5); -// } +TEST_CASE("GroupNormTest", "[ANNLayerTest]") +{ + arma::mat input, output, backwardOutput; + input = { + { 2, 0, 1 }, + { 3, 1, 2 }, + { 5, 1, 3 }, + { 7, 2, 4 }, + { 11, 3, 5 }, + { 13, 5, 6 }, + { 17, 8, 7 }, + { 19, 13, 8 } + }; + + GroupNorm<> model(2, 4); + model.Reset(); + + model.Forward(input, output); + arma::mat result; + result = { + { -1.1717001972, -1.4142135482, -1.3416407811 }, + { -0.6509445540, 0.0000000000 , -0.4472135937 }, + { 0.3905667324 , 0.0000000000 , 0.4472135937 }, + { 1.4320780188 , 1.4142135482 , 1.341640781 }, + { -1.2649110634, -1.1283296293, -1.3416407811 }, + { -0.6324555317, -0.5973509802, -0.4472135937 }, + { 0.6324555317 , 0.1991169934 , 0.4472135937 }, + { 1.2649110634 , 1.5265636161 , 1.3416407811 } + }; + + CheckMatrices(output, result, 1e-5); +} /** * GroupNorm layer numerical gradient test. */ -// TEST_CASE("GradientGroupNormTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(10, 256)), -// target(arma::zeros(1, 256)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add >(); -// model->Add >(10, 10); -// model->Add >(1, 10); -// model->Add >(10, 2); -// model->Add >(); -// } -// -// ~GradientFunction() -// { -// delete model; -// } -// -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 256, false); -// model->Gradient(model->Parameters(), 0, gradient, 256); -// return error; -// } -// -// arma::mat& Parameters() { return model->Parameters(); } -// -// FFN* model; -// arma::mat input, target; -// } function; -// -// REQUIRE(CheckGradient(function) <= 1e-4); -// } +TEST_CASE("GradientGroupNormTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randn(10, 256)), + target(arma::zeros(1, 256)) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 10); + model->Add >(1, 10); + model->Add >(10, 2); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 256, false); + model->Gradient(model->Parameters(), 0, gradient, 256); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + REQUIRE(CheckGradient(function) <= 1e-4); +} /** * Tests the LayerNorm layer. - * + */ TEST_CASE("LayerNormTest", "[ANNLayerTest]") { arma::mat input, output; @@ -3786,7 +3132,7 @@ TEST_CASE("LayerNormTest", "[ANNLayerTest]") { 4.9, 3.0 }, { 4.7, 3.2 } }; - LayerNorm model(input.n_rows); + LayerNorm<> model(input.n_rows); model.Reset(); model.Forward(input, output); @@ -3809,11 +3155,10 @@ TEST_CASE("LayerNormTest", "[ANNLayerTest]") CheckMatrices(output, result, 1e-1); } -*/ /** * LayerNorm layer numerical gradient test. - * + */ TEST_CASE("GradientLayerNormTest", "[ANNLayerTest]") { // Add function gradient instantiation. @@ -3823,13 +3168,14 @@ TEST_CASE("GradientLayerNormTest", "[ANNLayerTest]") input(arma::randn(10, 256)), target(arma::zeros(1, 256)) { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 10); - model->Add(10); - model->Add(10, 2); - model->Add(); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 10); + model->Add >(10); + model->Add >(10, 2); + model->Add >(); } ~GradientFunction() @@ -3839,79 +3185,77 @@ TEST_CASE("GradientLayerNormTest", "[ANNLayerTest]") double Gradient(arma::mat& gradient) const { - double error = model->Evaluate(model->Parameters(), 0, 16, false); - model->Gradient(model->Parameters(), 0, gradient, 16); + double error = model->Evaluate(model->Parameters(), 0, 256, false); + model->Gradient(model->Parameters(), 0, gradient, 256); return error; } arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, NguyenWidrowInitialization>* model; arma::mat input, target; } function; REQUIRE(CheckGradient(function) <= 1e-4); } -*/ /** * Test that the functions that can access the parameters of the * Layer Norm layer work. - * + */ TEST_CASE("LayerNormLayerParametersTest", "[ANNLayerTest]") { // Parameter order : size, eps. - LayerNorm layer(5, 1e-3); + LayerNorm<> layer(5, 1e-3); // Make sure we can get the parameters successfully. REQUIRE(layer.InSize() == 5); REQUIRE(layer.Epsilon() == 1e-3); } -*/ - -// /** -// * Test if the AddMerge layer is able to forward the -// * Forward/Backward/Gradient calls. -// */ -// TEST_CASE("AddMergeRunTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta, error; - -// AddMerge<> module(true, true); - -// Linear<>* linear = new Linear<>(10, 10); -// module.Add(linear); - -// linear->Parameters().randu(); -// linear->Reset(); - -// input = arma::zeros(10, 1); -// module.Forward(input, output); - -// double parameterSum = arma::accu(linear->Parameters().submat( -// 100, 0, linear->Parameters().n_elem - 1, 0)); - -// // Test the Backward function. -// module.Backward(input, input, delta); - -// // Clean up before we break, -// delete linear; - -// REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); -// REQUIRE(arma::accu(delta) == 0); -// } /** - * Test if the MultiplyMerge layer is able to forward the + * Test if the AddMerge layer is able to forward the * Forward/Backward/Gradient calls. - * -TEST_CASE("MultiplyMergeRunTest", "[ANNLayerTest]") + */ +TEST_CASE("AddMergeRunTest", "[ANNLayerTest]") { arma::mat output, input, delta, error; - MultiplyMerge module(true, true); + AddMerge<> module(true, true); - Linear* linear = new Linear(10, 10); + Linear<>* linear = new Linear<>(10, 10); + module.Add(linear); + + linear->Parameters().randu(); + linear->Reset(); + + input = arma::zeros(10, 1); + module.Forward(input, output); + + double parameterSum = arma::accu(linear->Parameters().submat( + 100, 0, linear->Parameters().n_elem - 1, 0)); + + // Test the Backward function. + module.Backward(input, input, delta); + + // Clean up before we break, + delete linear; + + REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); + REQUIRE(arma::accu(delta) == 0); +} + +/** + * Test if the MultiplyMerge layer is able to forward the + * Forward/Backward/Gradient calls. + */ +TEST_CASE("MultiplyMergeRunTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta, error; + + MultiplyMerge<> module(true, true); + + Linear<>* linear = new Linear<>(10, 10); module.Add(linear); linear->Parameters().randu(); @@ -3932,22 +3276,21 @@ TEST_CASE("MultiplyMergeRunTest", "[ANNLayerTest]") REQUIRE(parameterSum == Approx(arma::accu(output)).epsilon(1e-5)); REQUIRE(arma::accu(delta) == 0); } -*/ /** * Simple subview module test. - * + */ TEST_CASE("SimpleSubviewLayerTest", "[ANNLayerTest]") { arma::mat output, input, delta, outputMat; - Subview moduleRow(1, 10, 19); + Subview<> moduleRow(1, 10, 19); // Test the Forward function for a vector. input = arma::ones(20, 1); moduleRow.Forward(input, output); REQUIRE(output.n_rows == 10); - Subview moduleMat(4, 3, 6, 0, 2); + Subview<> moduleMat(4, 3, 6, 0, 2); // Test the Forward function for a matrix. input = arma::ones(20, 8); @@ -3960,48 +3303,46 @@ TEST_CASE("SimpleSubviewLayerTest", "[ANNLayerTest]") REQUIRE(accu(delta) == 160); REQUIRE(delta.n_rows == 20); } -*/ /** * Subview index test. - * + */ TEST_CASE("SubviewIndexTest", "[ANNLayerTest]") { arma::mat outputEnd, outputMid, outputStart, input, delta; input = arma::linspace(1, 20, 20); // Slicing from the initial indices. - Subview moduleStart(1, 0, 9); + Subview<> moduleStart(1, 0, 9); arma::mat subStart = arma::linspace(1, 10, 10); moduleStart.Forward(input, outputStart); CheckMatrices(outputStart, subStart); // Slicing from the mid indices. - Subview moduleMid(1, 6, 15); + Subview<> moduleMid(1, 6, 15); arma::mat subMid = arma::linspace(7, 16, 10); moduleMid.Forward(input, outputMid); CheckMatrices(outputMid, subMid); // Slicing from the end indices. - Subview moduleEnd(1, 10, 19); + Subview<> moduleEnd(1, 10, 19); arma::mat subEnd = arma::linspace(11, 20, 10); moduleEnd.Forward(input, outputEnd); CheckMatrices(outputEnd, subEnd); } -*/ /** * Subview batch test. - * + */ TEST_CASE("SubviewBatchTest", "[ANNLayerTest]") { arma::mat output, input, outputCol, outputMat, outputDef; // All rows selected. - Subview moduleCol(1, 0, 19); + Subview<> moduleCol(1, 0, 19); // Test with inSize 1. input = arma::ones(20, 8); @@ -4009,7 +3350,7 @@ TEST_CASE("SubviewBatchTest", "[ANNLayerTest]") CheckMatrices(outputCol, input); // Few rows and columns selected. - Subview moduleMat(4, 3, 6, 0, 2); + Subview<> moduleMat(4, 3, 6, 0, 2); // Test with inSize greater than 1. moduleMat.Forward(input, outputMat); @@ -4017,24 +3358,23 @@ TEST_CASE("SubviewBatchTest", "[ANNLayerTest]") CheckMatrices(outputMat, output); // endCol changed to 3 by default. - Subview moduleDef(4, 1, 6, 0, 4); + Subview<> moduleDef(4, 1, 6, 0, 4); // Test with inSize greater than 1 and endCol >= inSize. moduleDef.Forward(input, outputDef); output = arma::ones(24, 2); CheckMatrices(outputDef, output); } -*/ /** * Test that the functions that can modify and access the parameters of the * Subview layer work. - * + */ TEST_CASE("SubviewLayerParametersTest", "[ANNLayerTest]") { // Parameter order : inSize, beginRow, endRow, beginCol, endCol. - Subview layer1(1, 2, 3, 4, 5); - Subview layer2(1, 3, 4, 5, 6); + Subview<> layer1(1, 2, 3, 4, 5); + Subview<> layer2(1, 3, 4, 5, 6); // Make sure we can get the parameters correctly. REQUIRE(layer1.InSize() == 1); @@ -4056,15 +3396,14 @@ TEST_CASE("SubviewLayerParametersTest", "[ANNLayerTest]") REQUIRE(layer1.BeginCol() == layer2.BeginCol()); REQUIRE(layer1.EndCol() == layer2.EndCol()); } -*/ /* * Simple Reparametrization module test. - * + */ TEST_CASE("SimpleReparametrizationLayerTest", "[ANNLayerTest]") { arma::mat input, output, delta; - Reparametrization module(5); + Reparametrization<> module(5); // Test the Forward function. As the mean is zero and the standard // deviation is small, after multiplying the gaussian sample, the @@ -4079,15 +3418,14 @@ TEST_CASE("SimpleReparametrizationLayerTest", "[ANNLayerTest]") module.Backward(input, gy, delta); REQUIRE(arma::accu(delta) != 0); // klBackward will be added. } -*/ /** * Reparametrization module stochastic boolean test. - * + */ TEST_CASE("ReparametrizationLayerStochasticTest", "[ANNLayerTest]") { arma::mat input, outputA, outputB; - Reparametrization module(5, false); + Reparametrization<> module(5, false); input = join_cols(arma::ones(5, 1), arma::zeros(5, 1)); @@ -4098,15 +3436,14 @@ TEST_CASE("ReparametrizationLayerStochasticTest", "[ANNLayerTest]") CheckMatrices(outputA, outputB); } -*/ /** * Reparametrization module includeKl boolean test. - * + */ TEST_CASE("ReparametrizationLayerIncludeKlTest", "[ANNLayerTest]") { arma::mat input, output, gy, delta; - Reparametrization module(5, true, false); + Reparametrization<> module(5, true, false); input = join_cols(arma::ones(5, 1), arma::zeros(5, 1)); @@ -4119,31 +3456,29 @@ TEST_CASE("ReparametrizationLayerIncludeKlTest", "[ANNLayerTest]") REQUIRE(arma::accu(delta) == 0); } -*/ /** * Jacobian Reparametrization module test. - * + */ TEST_CASE("JacobianReparametrizationLayerTest", "[ANNLayerTest]") { for (size_t i = 0; i < 5; ++i) { - const size_t inputElementsHalf = math::RandInt(2, 10); + const size_t inputElementsHalf = math::RandInt(2, 1000); arma::mat input; input.set_size(inputElementsHalf * 2, 1); - Reparametrization module(inputElementsHalf, false, false); + Reparametrization<> module(inputElementsHalf, false, false); double error = JacobianTest(module, input); REQUIRE(error <= 1e-5); } } -*/ /** * Reparametrization layer numerical gradient test. - * + */ TEST_CASE("GradientReparametrizationLayerTest", "[ANNLayerTest]") { // Linear function gradient instantiation. @@ -4153,13 +3488,14 @@ TEST_CASE("GradientReparametrizationLayerTest", "[ANNLayerTest]") input(arma::randu(10, 1)), target(arma::mat("0")) { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 6); - model->Add(3, false, true, 1); - model->Add(3, 2); - model->Add(); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 6); + model->Add >(3, false, true, 1); + model->Add >(3, 2); + model->Add >(); } ~GradientFunction() @@ -4176,17 +3512,16 @@ TEST_CASE("GradientReparametrizationLayerTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, NguyenWidrowInitialization>* model; arma::mat input, target; } function; - // REQUIRE(CheckGradient(function) <= 1e-4); + REQUIRE(CheckGradient(function) <= 1e-4); } -*/ /** * Reparametrization layer beta numerical gradient test. - * + */ TEST_CASE("GradientReparametrizationLayerBetaTest", "[ANNLayerTest]") { // Linear function gradient instantiation. @@ -4196,14 +3531,15 @@ TEST_CASE("GradientReparametrizationLayerBetaTest", "[ANNLayerTest]") input(arma::randu(10, 2)), target(arma::mat("0 0")) { - model = new FFN(); - model->ResetData(input, target); - model->Add(); - model->Add(10, 6); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 6); // Use a value of beta not equal to 1. - model->Add(3, false, true, 2); - model->Add(3, 2); - model->Add(); + model->Add >(3, false, true, 2); + model->Add >(3, 2); + model->Add >(); } ~GradientFunction() @@ -4220,22 +3556,21 @@ TEST_CASE("GradientReparametrizationLayerBetaTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, NguyenWidrowInitialization>* model; arma::mat input, target; } function; - // REQUIRE(CheckGradient(function) <= 1e-4); + REQUIRE(CheckGradient(function) <= 1e-4); } -*/ /** * Test that the functions that can access the parameters of the * Reparametrization layer work. - * + */ TEST_CASE("ReparametrizationLayerParametersTest", "[ANNLayerTest]") { // Parameter order : latentSize, stochastic, includeKL, beta. - Reparametrization layer(5, false, false, 2); + Reparametrization<> layer(5, false, false, 2); // Make sure we can get the parameters successfully. REQUIRE(layer.OutputSize() == 5); @@ -4243,22 +3578,21 @@ TEST_CASE("ReparametrizationLayerParametersTest", "[ANNLayerTest]") REQUIRE(layer.IncludeKL() == false); REQUIRE(layer.Beta() == 2); } -*/ /** * Simple residual module test. - * + */ TEST_CASE("SimpleResidualLayerTest", "[ANNLayerTest]") { arma::mat outputA, outputB, input, deltaA, deltaB; - Sequential* sequential = new Sequential(true); - Residual* residual = new Residual(true); + Sequential<>* sequential = new Sequential<>(true); + Residual<>* residual = new Residual<>(true); - Linear* linearA = new Linear(10, 10); + Linear<>* linearA = new Linear<>(10, 10); linearA->Parameters().randu(); linearA->Reset(); - Linear* linearB = new Linear(10, 10); + Linear<>* linearB = new Linear<>(10, 10); linearB->Parameters().randu(); linearB->Reset(); @@ -4288,23 +3622,22 @@ TEST_CASE("SimpleResidualLayerTest", "[ANNLayerTest]") delete linearA; delete linearB; } -*/ /** * Simple Highway module test. - * + */ TEST_CASE("SimpleHighwayLayerTest", "[ANNLayerTest]") { arma::mat outputA, outputB, input, deltaA, deltaB; - Sequential* sequential = new Sequential(true); - Highway* highway = new Highway(10, true); + Sequential<>* sequential = new Sequential<>(true); + Highway<>* highway = new Highway<>(10, true); highway->Parameters().zeros(); highway->Reset(); - Linear* linearA = new Linear(10, 10); + Linear<>* linearA = new Linear<>(10, 10); linearA->Parameters().randu(); linearA->Reset(); - Linear* linearB = new Linear(10, 10); + Linear<>* linearB = new Linear<>(10, 10); linearB->Parameters().randu(); linearB->Reset(); @@ -4327,247 +3660,249 @@ TEST_CASE("SimpleHighwayLayerTest", "[ANNLayerTest]") delete linearA; delete linearB; } -*/ /** * Test that the function that can access the inSize parameter of the * Highway layer works. - * + */ TEST_CASE("HighwayLayerParametersTest", "[ANNLayerTest]") { // Parameter order : inSize, model. - Highway layer(1, true); + Highway<> layer(1, true); // Make sure we can get the parameter successfully. REQUIRE(layer.InSize() == 1); } -*/ - -// /** -// * Sequential layer numerical gradient test. -// */ -// TEST_CASE("GradientHighwayLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(5, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(); -// model->Add(5, 10); - -// highway = new Highway(10); -// highway->Add(10, 10); -// highway->Add(); -// highway->Add(10, 10); -// highway->Add(); - -// model->Add(highway); -// model->Add(10, 2); -// model->Add(); -// } - -// ~GradientFunction() -// { -// delete model; -// } - -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } - -// arma::mat& Parameters() { return model->Parameters(); } - -// FFN* model; -// Highway* highway; -// arma::mat input, target; -// } function; - -// REQUIRE(CheckGradient(function) <= 1e-4); -// } /** * Sequential layer numerical gradient test. */ -// TEST_CASE("GradientSequentialLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(); -// model->Add(10, 10); -// sequential = new Sequential(); -// sequential->Add(10, 10); -// sequential->Add(); -// sequential->Add(10, 5); -// sequential->Add(); +TEST_CASE("GradientHighwayLayerTest", "[ANNLayerTest]") +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(5, 1)), + target(arma::mat("0")) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(5, 10); -// model->Add(sequential); -// model->Add(5, 2); -// model->Add(); -// } + highway = new Highway<>(10); + highway->Add >(10, 10); + highway->Add >(); + highway->Add >(10, 10); + highway->Add >(); -// ~GradientFunction() -// { -// delete model; -// } + model->Add(highway); + model->Add >(10, 2); + model->Add >(); + } -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } + ~GradientFunction() + { + delete model; + } -// arma::mat& Parameters() { return model->Parameters(); } + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } -// FFN* model; -// Sequential* sequential; -// arma::mat input, target; -// } function; + arma::mat& Parameters() { return model->Parameters(); } -// REQUIRE(CheckGradient(function) <= 1e-4); -// } + FFN, NguyenWidrowInitialization>* model; + Highway<>* highway; + arma::mat input, target; + } function; -// /** -// * WeightNorm layer numerical gradient test. -// */ -// TEST_CASE("GradientWeightNormLayerTest", "[ANNLayerTest]") -// { -// // Linear function gradient instantiation. -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randu(10, 1)), -// target(arma::mat("0")) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add(10, 10); + REQUIRE(CheckGradient(function) <= 1e-4); +} -// Linear* linear = new Linear(10, 2); -// weightNorm = new WeightNorm(linear); +/** + * Sequential layer numerical gradient test. + */ +TEST_CASE("GradientSequentialLayerTest", "[ANNLayerTest]") +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 10); + sequential = new Sequential<>(); + sequential->Add >(10, 10); + sequential->Add >(); + sequential->Add >(10, 5); + sequential->Add >(); -// model->Add(weightNorm); -// model->Add(); -// } + model->Add(sequential); + model->Add >(5, 2); + model->Add >(); + } -// ~GradientFunction() -// { -// delete model; -// } + ~GradientFunction() + { + delete model; + } -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1); -// model->Gradient(model->Parameters(), 0, gradient, 1); -// return error; -// } + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } -// arma::mat& Parameters() { return model->Parameters(); } + arma::mat& Parameters() { return model->Parameters(); } -// FFN* model; -// WeightNorm* weightNorm; -// arma::mat input, target; -// } function; + FFN, NguyenWidrowInitialization>* model; + Sequential<>* sequential; + arma::mat input, target; + } function; -// REQUIRE(CheckGradient(function) <= 1e-4); -// } + REQUIRE(CheckGradient(function) <= 1e-4); +} -// /** -// * Test if the WeightNorm layer is able to forward the -// * Forward/Backward/Gradient calls. -// */ -// TEST_CASE("WeightNormRunTest", "[ANNLayerTest]") -// { -// arma::mat output, input, delta, error; -// Linear* linear = new Linear(10, 10); +/** + * WeightNorm layer numerical gradient test. + */ +TEST_CASE("GradientWeightNormLayerTest", "[ANNLayerTest]") +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() : + input(arma::randu(10, 1)), + target(arma::mat("0")) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(10, 10); -// WeightNorm module(linear); + Linear<>* linear = new Linear<>(10, 2); + weightNorm = new WeightNorm<>(linear); -// module.Parameters().randu(); -// module.Reset(); + model->Add(weightNorm); + model->Add >(); + } -// linear->Bias().zeros(); + ~GradientFunction() + { + delete model; + } -// input = arma::zeros(10, 1); -// module.Forward(input, output); + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } -// // Test the Backward function. -// module.Backward(input, input, delta); + arma::mat& Parameters() { return model->Parameters(); } -// REQUIRE(0 == arma::accu(output)); -// REQUIRE(arma::accu(delta) == 0); -// } + FFN, NguyenWidrowInitialization>* model; + WeightNorm<>* weightNorm; + arma::mat input, target; + } function; -// // General ANN serialization test. -// template -// void ANNLayerSerializationTest(LayerType& layer) -// { -// arma::mat input(5, 100, arma::fill::randu); -// arma::mat output(5, 100, arma::fill::randu); + REQUIRE(CheckGradient(function) <= 1e-4); +} -// FFN model; -// model.Add>(input.n_rows, 10); -// model.Add(layer); -// model.Add>(); -// model.Add>(10, output.n_rows); -// model.Add>(); +/** + * Test if the WeightNorm layer is able to forward the + * Forward/Backward/Gradient calls. + */ +TEST_CASE("WeightNormRunTest", "[ANNLayerTest]") +{ + arma::mat output, input, delta, error; -// ens::StandardSGD opt(0.1, 1, 5, -100, false); -// model.Train(input, output, opt); + Linear<>* linear = new Linear<>(10, 10); -// arma::mat originalOutput; -// model.Predict(input, originalOutput); + WeightNorm<> module(linear); -// // Now serialize the model. -// FFN xmlModel, jsonModel, -// binaryModel; -// SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); + module.Parameters().randu(); + module.Reset(); -// // Ensure that predictions are the same. -// arma::mat modelOutput, xmlOutput, jsonOutput, binaryOutput; -// model.Predict(input, modelOutput); -// xmlModel.Predict(input, xmlOutput); -// jsonModel.Predict(input, jsonOutput); -// binaryModel.Predict(input, binaryOutput); + linear->Bias().zeros(); -// CheckMatrices(originalOutput, modelOutput, 1e-5); -// CheckMatrices(originalOutput, xmlOutput, 1e-5); -// CheckMatrices(originalOutput, jsonOutput, 1e-5); -// CheckMatrices(originalOutput, binaryOutput, 1e-5); -// } + input = arma::zeros(10, 1); + module.Forward(input, output); -// /** -// * Simple serialization test for batch normalization layer. -// */ -// TEST_CASE("BatchNormSerializationTest", "[ANNLayerTest]") -// { -// BatchNorm<> layer(10); -// ANNLayerSerializationTest(layer); -// } + // Test the Backward function. + module.Backward(input, input, delta); -// /** -// * Simple serialization test for layer normalization layer. -// */ -// TEST_CASE("LayerNormSerializationTest", "[ANNLayerTest]") -// { -// LayerNorm<> layer(10); -// ANNLayerSerializationTest(layer); -// } + REQUIRE(0 == arma::accu(output)); + REQUIRE(arma::accu(delta) == 0); +} + +// General ANN serialization test. +template +void ANNLayerSerializationTest(LayerType& layer) +{ + arma::mat input(5, 100, arma::fill::randu); + arma::mat output(5, 100, arma::fill::randu); + + FFN, ann::RandomInitialization> model; + model.Add>(input.n_rows, 10); + model.Add(layer); + model.Add>(); + model.Add>(10, output.n_rows); + model.Add>(); + + ens::StandardSGD opt(0.1, 1, 5, -100, false); + model.Train(input, output, opt); + + arma::mat originalOutput; + model.Predict(input, originalOutput); + + // Now serialize the model. + FFN, ann::RandomInitialization> xmlModel, jsonModel, + binaryModel; + SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); + + // Ensure that predictions are the same. + arma::mat modelOutput, xmlOutput, jsonOutput, binaryOutput; + model.Predict(input, modelOutput); + xmlModel.Predict(input, xmlOutput); + jsonModel.Predict(input, jsonOutput); + binaryModel.Predict(input, binaryOutput); + + CheckMatrices(originalOutput, modelOutput, 1e-5); + CheckMatrices(originalOutput, xmlOutput, 1e-5); + CheckMatrices(originalOutput, jsonOutput, 1e-5); + CheckMatrices(originalOutput, binaryOutput, 1e-5); +} + +/** + * Simple serialization test for batch normalization layer. + */ +TEST_CASE("BatchNormSerializationTest", "[ANNLayerTest]") +{ + BatchNorm<> layer(10); + ANNLayerSerializationTest(layer); +} + +/** + * Simple serialization test for layer normalization layer. + */ +TEST_CASE("LayerNormSerializationTest", "[ANNLayerTest]") +{ + LayerNorm<> layer(10); + ANNLayerSerializationTest(layer); +} /** * Test that the functions that can modify and access the parameters of the @@ -4575,13 +3910,16 @@ TEST_CASE("HighwayLayerParametersTest", "[ANNLayerTest]") */ TEST_CASE("ConvolutionLayerParametersTest", "[ANNLayerTest]") { - // Parameter order: outSize, kW, kH, dW, dH, padW, padH, paddingType. - Convolution layer1(2, 3, 4, 5, 6, std::tuple(7, 8), - std::tuple(9, 10), "none"); - Convolution layer2(3, 4, 5, 6, 7, std::tuple(8, 9), - std::tuple(10, 11), "none"); + // Parameter order: inSize, outSize, kW, kH, dW, dH, padW, padH, inputWidth, + // inputHeight, paddingType. + Convolution<> layer1(1, 2, 3, 4, 5, 6, std::tuple(7, 8), + std::tuple(9, 10), 11, 12, "none"); + Convolution<> layer2(2, 3, 4, 5, 6, 7, std::tuple(8, 9), + std::tuple(10, 11), 12, 13, "none"); // Make sure we can get the parameters successfully. + REQUIRE(layer1.InputWidth() == 11); + REQUIRE(layer1.InputHeight() == 12); REQUIRE(layer1.KernelWidth() == 3); REQUIRE(layer1.KernelHeight() == 4); REQUIRE(layer1.StrideWidth() == 5); @@ -4592,6 +3930,8 @@ TEST_CASE("ConvolutionLayerParametersTest", "[ANNLayerTest]") REQUIRE(layer1.PadHBottom() == 10); // Now modify the parameters to match the second layer. + layer1.InputWidth() = 12; + layer1.InputHeight() = 13; layer1.KernelWidth() = 4; layer1.KernelHeight() = 5; layer1.StrideWidth() = 6; @@ -4602,6 +3942,8 @@ TEST_CASE("ConvolutionLayerParametersTest", "[ANNLayerTest]") layer1.PadHBottom() = 11; // Now ensure all results are the same. + REQUIRE(layer1.InputWidth() == layer2.InputWidth()); + REQUIRE(layer1.InputHeight() == layer2.InputHeight()); REQUIRE(layer1.KernelWidth() == layer2.KernelWidth()); REQUIRE(layer1.KernelHeight() == layer2.KernelHeight()); REQUIRE(layer1.StrideWidth() == layer2.StrideWidth()); @@ -4620,18 +3962,13 @@ TEST_CASE("ConvolutionLayerPaddingTest", "[ANNLayerTest]") arma::mat output, input, delta; // Check valid padding option. - Convolution module1(1, 3, 3, 1, 1, std::tuple(1, 1), - std::tuple(1, 1), "valid"); - module1.InputDimensions() = std::vector({ 7, 7 }); - module1.ComputeOutputDimensions(); - arma::mat weights1(module1.WeightSize(), 1); - REQUIRE(weights1.n_elem == 10); - module1.SetWeights(weights1.memptr()); + Convolution<> module1(1, 1, 3, 3, 1, 1, std::tuple(1, 1), + std::tuple(1, 1), 7, 7, "valid"); // Test the Forward function. input = arma::linspace(0, 48, 49); - output.set_size(module1.OutputSize(), 1); - module1.Parameters().zeros(); + module1.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module1.Reset(); module1.Forward(input, output); REQUIRE(arma::accu(output) == 0); @@ -4639,22 +3976,16 @@ TEST_CASE("ConvolutionLayerPaddingTest", "[ANNLayerTest]") REQUIRE(output.n_cols == 1); // Test the Backward function. - delta.set_size(arma::size(input)); module1.Backward(input, output, delta); // Check same padding option. - Convolution module2(1, 3, 3, 1, 1, std::tuple(0, 0), - std::tuple(0, 0), "same"); - module2.InputDimensions() = std::vector({ 7, 7 }); - module2.ComputeOutputDimensions(); - arma::mat weights2(module2.WeightSize(), 1); - REQUIRE(weights2.n_elem == 10); - module2.SetWeights(weights2.memptr()); + Convolution<> module2(1, 1, 3, 3, 1, 1, std::tuple(0, 0), + std::tuple(0, 0), 7, 7, "same"); // Test the forward function. input = arma::linspace(0, 48, 49); - output.set_size(module2.OutputSize(), 1); - module2.Parameters().zeros(); + module2.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); + module2.Reset(); module2.Forward(input, output); REQUIRE(arma::accu(output) == 0); @@ -4662,59 +3993,17 @@ TEST_CASE("ConvolutionLayerPaddingTest", "[ANNLayerTest]") REQUIRE(output.n_cols == 1); // Test the backward function. - delta.set_size(arma::size(input)); module2.Backward(input, output, delta); } -/** - * Convolution layer numerical gradient test. - */ -TEST_CASE("GradientConvolutionLayerTest", "[ANNLayerTest]") -{ - struct GradientFunction - { - GradientFunction() : - input(arma::linspace(0, 35, 36)), - target(arma::mat("1")) - { - model = new FFN(); - model->ResetData(input, target); - model->Add(1, 3, 3, 1, 1, std::tuple(0, 0), - std::tuple(0, 0), "same"); - model->Add(); - - model->InputDimensions() = std::vector({ 6, 6 }); - } - - ~GradientFunction() - { - delete model; - } - - double Gradient(arma::mat& gradient) const - { - double error = model->Evaluate(model->Parameters(), 0, 1); - model->Gradient(model->Parameters(), 0, gradient, 1); - return error; - } - - arma::mat& Parameters() { return model->Parameters(); } - - FFN* model; - arma::mat input, target; - } function; - - REQUIRE(CheckGradient(function) < 1e3); -} - /** * Test that the padding options in Transposed Convolution layer. - * + */ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") { arma::mat output, input, delta; - TransposedConvolution module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6, "VALID"); + TransposedConvolution<> module1(1, 1, 3, 3, 1, 1, 0, 0, 4, 4, 6, 6, "VALID"); // Test the forward function. // Valid Should give the same result. input = arma::linspace(0, 15, 16); @@ -4729,7 +4018,7 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") REQUIRE(arma::accu(delta) == 0.0); // Test Valid for non zero padding. - TransposedConvolution module2(1, 1, 3, 3, 2, 2, + TransposedConvolution<> module2(1, 1, 3, 3, 2, 2, std::tuple(0, 0), std::tuple(0, 0), 2, 2, 5, 5, "VALID"); // Test the forward function. @@ -4749,7 +4038,7 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") REQUIRE(arma::accu(delta) == 960.0); // Test for same padding type. - TransposedConvolution module3(1, 1, 3, 3, 2, 2, 0, 0, 3, 3, 3, 3, "SAME"); + TransposedConvolution<> module3(1, 1, 3, 3, 2, 2, 0, 0, 3, 3, 3, 3, "SAME"); // Test the forward function. input = arma::linspace(0, 8, 9); module3.Parameters() = arma::mat(9 + 1, 1, arma::fill::zeros); @@ -4764,7 +4053,7 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") REQUIRE(arma::accu(delta) == 0.0); // Output shape should equal input. - TransposedConvolution module4(1, 1, 3, 3, 1, 1, + TransposedConvolution<> module4(1, 1, 3, 3, 1, 1, std::tuple(2, 2), std::tuple(2, 2), 5, 5, 5, 5, "SAME"); // Test the forward function. @@ -4780,7 +4069,7 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") module4.Backward(input, output, delta); REQUIRE(arma::accu(delta) == 0.0); - TransposedConvolution module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 2, 2, "SAME"); + TransposedConvolution<> module5(1, 1, 3, 3, 2, 2, 0, 0, 2, 2, 2, 2, "SAME"); // Test the forward function. input = arma::linspace(0, 3, 4); module5.Parameters() = arma::mat(25 + 1, 1, arma::fill::zeros); @@ -4794,7 +4083,7 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") module5.Backward(input, output, delta); REQUIRE(arma::accu(delta) == 0.0); - TransposedConvolution module6(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 5, 5, "SAME"); + TransposedConvolution<> module6(1, 1, 4, 4, 1, 1, 1, 1, 5, 5, 5, 5, "SAME"); // Test the forward function. input = arma::linspace(0, 24, 25); module6.Parameters() = arma::mat(16 + 1, 1, arma::fill::zeros); @@ -4808,103 +4097,102 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") module6.Backward(input, output, delta); REQUIRE(arma::accu(delta) == 0.0); } -*/ /** * Simple test for Lp Pooling layer. */ -// TEST_CASE("LpMaxPoolingTestCase", "[ANNLayerTest]") -// { -// // For rectangular input to pooling layers. -// arma::mat input = arma::mat(8, 1); -// arma::mat output; -// input.zeros(); -// input(0) = input(6) = 30; -// input(1) = input(7) = 120; -// input(2) = input(4) = 272; -// input(3) = input(5) = 315; -// // Output-Size should be 1 x 2. -// // Square output. -// LpPooling<> module1(4, 2, 2, 2, 2); -// module1.InputHeight() = 2; -// module1.InputWidth() = 4; -// module1.Forward(input, output); -// // Calculated using torch.nn.LPPool2d(). -// REQUIRE(arma::accu(output) - 706.0 == Approx(0.0).margin(2e-5)); -// REQUIRE(output.n_elem == 2); -// -// // For Square input. -// input = arma::mat(16, 1); -// input.zeros(); -// input(0) = 4; -// input(1) = 3; -// input(3) = 12; -// input(7) = 35; -// input(8) = 6; -// input(11) = 7; -// input(12) = 8; -// input(15) = 24; -// // Output-Size should be 2 x 2. -// // Square output. -// LpPooling<> module3(2, 2, 2, 2, 2); -// module3.InputHeight() = 4; -// module3.InputWidth() = 4; -// module3.Forward(input, output); -// // Calculated using torch.nn.LPPool2d(). -// REQUIRE(arma::accu(output) - 77.0 == Approx(0.0).margin(2e-5)); -// REQUIRE(output.n_elem == 4); -// } +TEST_CASE("LpMaxPoolingTestCase", "[ANNLayerTest]") +{ + // For rectangular input to pooling layers. + arma::mat input = arma::mat(8, 1); + arma::mat output; + input.zeros(); + input(0) = input(6) = 30; + input(1) = input(7) = 120; + input(2) = input(4) = 272; + input(3) = input(5) = 315; + // Output-Size should be 1 x 2. + // Square output. + LpPooling<> module1(4, 2, 2, 2, 2); + module1.InputHeight() = 2; + module1.InputWidth() = 4; + module1.Forward(input, output); + // Calculated using torch.nn.LPPool2d(). + REQUIRE(arma::accu(output) - 706.0 == Approx(0.0).margin(2e-5)); + REQUIRE(output.n_elem == 2); + + // For Square input. + input = arma::mat(16, 1); + input.zeros(); + input(0) = 4; + input(1) = 3; + input(3) = 12; + input(7) = 35; + input(8) = 6; + input(11) = 7; + input(12) = 8; + input(15) = 24; + // Output-Size should be 2 x 2. + // Square output. + LpPooling<> module3(2, 2, 2, 2, 2); + module3.InputHeight() = 4; + module3.InputWidth() = 4; + module3.Forward(input, output); + // Calculated using torch.nn.LPPool2d(). + REQUIRE(arma::accu(output) - 77.0 == Approx(0.0).margin(2e-5)); + REQUIRE(output.n_elem == 4); +} /** * Simple test for Mean Pooling layer. */ -// TEST_CASE("MeanPoolingTestCase", "[ANNLayerTest]") -// { -// // For rectangular input to pooling layers. -// arma::mat input = arma::mat(28, 1); -// input.zeros(); -// input(0) = input(16) = 1; -// input(1) = input(17) = 2; -// input(2) = input(18) = 3; -// input(3) = input(19) = 4; -// input(4) = input(20) = 5; -// input(5) = input(23) = 6; -// input(6) = input(24) = 7; -// input(14) = input(25) = 8; -// input(15) = input(26) = 9; -// -// MeanPooling<> module1(2, 2, 2, 2, false); -// MeanPooling<> module2(2, 2, 2, 2, true); -// module1.InputWidth() = 7; -// module1.InputHeight() = 4; -// module2.InputWidth() = 7; -// module2.InputHeight() = 4; -// -// // Calculated using torch.nn.MeanPool2d(). -// arma::mat result1, result2; -// result1 << 0.7500 << 4.2500 << arma::endr -// << 1.7500 << 4.0000 << arma::endr -// << 2.7500 << 6.0000 << arma::endr -// << 3.5000 << 2.5000 << arma::endr; -// -// result2 << 0.7500 << 4.2500 << arma::endr -// << 1.7500 << 4.0000 << arma::endr -// << 2.7500 << 6.0000 << arma::endr; -// -// arma::mat output1, output2; -// module1.Forward(input, output1); -// module2.Forward(input, output2); -// output1.reshape(4, 2); -// output2.reshape(3, 2); -// CheckMatrices(output1, result1, 1e-1); -// CheckMatrices(output2, result2, 1e-1); -// -// arma::mat delta1, delta2; -// module1.Backward(input, output1, delta1); -// REQUIRE(arma::accu(delta1) == 25.5); -// module2.Backward(input, output2, delta2); -// REQUIRE(arma::accu(delta2) == 19.5); -// } +TEST_CASE("MeanPoolingTestCase", "[ANNLayerTest]") +{ + // For rectangular input to pooling layers. + arma::mat input = arma::mat(28, 1); + input.zeros(); + input(0) = input(16) = 1; + input(1) = input(17) = 2; + input(2) = input(18) = 3; + input(3) = input(19) = 4; + input(4) = input(20) = 5; + input(5) = input(23) = 6; + input(6) = input(24) = 7; + input(14) = input(25) = 8; + input(15) = input(26) = 9; + + MeanPooling<> module1(2, 2, 2, 2, false); + MeanPooling<> module2(2, 2, 2, 2, true); + module1.InputWidth() = 7; + module1.InputHeight() = 4; + module2.InputWidth() = 7; + module2.InputHeight() = 4; + + // Calculated using torch.nn.MeanPool2d(). + arma::mat result1, result2; + result1 << 0.7500 << 4.2500 << arma::endr + << 1.7500 << 4.0000 << arma::endr + << 2.7500 << 6.0000 << arma::endr + << 3.5000 << 2.5000 << arma::endr; + + result2 << 0.7500 << 4.2500 << arma::endr + << 1.7500 << 4.0000 << arma::endr + << 2.7500 << 6.0000 << arma::endr; + + arma::mat output1, output2; + module1.Forward(input, output1); + module2.Forward(input, output2); + output1.reshape(4, 2); + output2.reshape(3, 2); + CheckMatrices(output1, result1, 1e-1); + CheckMatrices(output2, result2, 1e-1); + + arma::mat delta1, delta2; + module1.Backward(input, output1, delta1); + REQUIRE(arma::accu(delta1) == 25.5); + module2.Backward(input, output2, delta2); + REQUIRE(arma::accu(delta2) == 19.5); +} /** * Simple test for Max Pooling layer. @@ -4925,12 +4213,10 @@ TEST_CASE("MaxPoolingTestCase", "[ANNLayerTest]") input(10) = 8; input(11) = 9; // Output-Size should be 2 x 2. - output.set_size(4, 1); - // Square output. - MaxPooling module1(2, 2, 2, 1); - module1.InputDimensions() = std::vector({ 4, 3 }); - module1.ComputeOutputDimensions(); + MaxPooling<> module1(2, 2, 2, 1); + module1.InputHeight() = 3; + module1.InputWidth() = 4; module1.Forward(input, output); // Calculated using torch.nn.MaxPool2d(). REQUIRE(arma::accu(output) == 28); @@ -4946,12 +4232,10 @@ TEST_CASE("MaxPoolingTestCase", "[ANNLayerTest]") input(3) = 3; input(6) = 3; // Output-Size should be 1 x 2. - output.set_size(2, 1); - // Rectangular output. - MaxPooling module2(3, 2, 3, 1); - module2.InputDimensions() = std::vector({ 3, 3 }); - module2.ComputeOutputDimensions(); + MaxPooling<> module2(3, 2, 3, 1); + module2.InputHeight() = 3; + module2.InputWidth() = 3; module2.Forward(input, output); // Calculated using torch.nn.MaxPool2d(). REQUIRE(arma::accu(output) == 12.0); @@ -4967,12 +4251,10 @@ TEST_CASE("MaxPoolingTestCase", "[ANNLayerTest]") input(4) = 3; input(8) = 3; // Output-Size should be 3 x 3. - output.set_size(9, 1); - // Square output. - MaxPooling module3(2, 2, 1, 1); - module3.InputDimensions() = std::vector({ 4, 4 }); - module3.ComputeOutputDimensions(); + MaxPooling<> module3(2, 2, 1, 1); + module3.InputHeight() = 4; + module3.InputWidth() = 4; module3.Forward(input, output); // Calculated using torch.nn.MaxPool2d(). REQUIRE(arma::accu(output) == 30.0); @@ -4986,12 +4268,10 @@ TEST_CASE("MaxPoolingTestCase", "[ANNLayerTest]") input(1) = 1; input(3) = 1; // Output-Size should be 2 x 2. - output.set_size(4, 1); - // Square output. - MaxPooling module4(2, 1, 1, 1); - module4.InputDimensions() = std::vector({ 3, 2 }); - module4.ComputeOutputDimensions(); + MaxPooling<> module4(2, 1, 1, 1); + module4.InputHeight() = 2; + module4.InputWidth() = 3; module4.Forward(input, output); // Calculated using torch.nn.MaxPool2d(). REQUIRE(arma::accu(output) == 3); @@ -5002,12 +4282,12 @@ TEST_CASE("MaxPoolingTestCase", "[ANNLayerTest]") /** * Test that the functions that can modify and access the parameters of the * Glimpse layer work. - * + */ TEST_CASE("GlimpseLayerParametersTest", "[ANNLayerTest]") { // Parameter order : inSize, size, depth, scale, inputWidth, inputHeight. - Glimpse layer1(1, 2, 3, 4, 5, 6); - Glimpse layer2(1, 2, 3, 4, 6, 7); + Glimpse<> layer1(1, 2, 3, 4, 5, 6); + Glimpse<> layer2(1, 2, 3, 4, 6, 7); // Make sure we can get the parameters successfully. REQUIRE(layer1.InputHeight() == 6); @@ -5029,25 +4309,37 @@ TEST_CASE("GlimpseLayerParametersTest", "[ANNLayerTest]") REQUIRE(layer1.GlimpseSize() == layer2.GlimpseSize()); REQUIRE(layer1.InSize() == layer2.InSize()); } -*/ /** * Test that the function that can access the stdev parameter of the * Reinforce Normal layer works. - * + */ TEST_CASE("ReinforceNormalLayerParametersTest", "[ANNLayerTest]") { // Parameter : stdev. - ReinforceNormal layer(4.0); + ReinforceNormal<> layer(4.0); // Make sure we can get the parameter successfully. REQUIRE(layer.StandardDeviation() == 4.0); } -*/ + +/** + * Test that the function that can access the parameters of the + * VR Class Reward layer works. + */ +TEST_CASE("VRClassRewardLayerParametersTest", "[ANNLayerTest]") +{ + // Parameter order : scale, sizeAverage. + VRClassReward<> layer(2, false); + + // Make sure we can get the parameters successfully. + REQUIRE(layer.Scale() == 2); + REQUIRE(layer.SizeAverage() == false); +} /** * Simple test for Adaptive pooling for Max Pooling layer. - * + */ TEST_CASE("AdaptiveMaxPoolingTestCase", "[ANNLayerTest]") { // For rectangular input. @@ -5066,7 +4358,7 @@ TEST_CASE("AdaptiveMaxPoolingTestCase", "[ANNLayerTest]") input(11) = 9; // Output-Size should be 2 x 2. // Square output. - AdaptiveMaxPooling module1(2, 2); + AdaptiveMaxPooling<> module1(2, 2); module1.InputHeight() = 3; module1.InputWidth() = 4; module1.Forward(input, output); @@ -5088,7 +4380,7 @@ TEST_CASE("AdaptiveMaxPoolingTestCase", "[ANNLayerTest]") input(6) = 3; // Output-Size should be 1 x 2. // Rectangular output. - AdaptiveMaxPooling module2(2, 1); + AdaptiveMaxPooling<> module2(2, 1); module2.InputHeight() = 3; module2.InputWidth() = 3; module2.Forward(input, output); @@ -5110,7 +4402,7 @@ TEST_CASE("AdaptiveMaxPoolingTestCase", "[ANNLayerTest]") input(8) = 3; // Output-Size should be 3 x 3. // Square output. - AdaptiveMaxPooling module3(std::tuple(3, 3)); + AdaptiveMaxPooling<> module3(std::tuple(3, 3)); module3.InputHeight() = 4; module3.InputWidth() = 4; module3.Forward(input, output); @@ -5130,7 +4422,7 @@ TEST_CASE("AdaptiveMaxPoolingTestCase", "[ANNLayerTest]") input(3) = 1; // Output-Size should be 2 x 2. // Square output. - AdaptiveMaxPooling module4(std::tuple(2, 2)); + AdaptiveMaxPooling<> module4(std::tuple(2, 2)); module4.InputHeight() = 4; module4.InputWidth() = 5; module4.Forward(input, output); @@ -5142,11 +4434,10 @@ TEST_CASE("AdaptiveMaxPoolingTestCase", "[ANNLayerTest]") module4.Backward(input, output, delta); REQUIRE(arma::accu(delta) == 2.0); } -*/ /** * Simple test for Adaptive pooling for Mean Pooling layer. - * + */ TEST_CASE("AdaptiveMeanPoolingTestCase", "[ANNLayerTest]") { // For rectangular input. @@ -5165,7 +4456,7 @@ TEST_CASE("AdaptiveMeanPoolingTestCase", "[ANNLayerTest]") input(11) = 9; // Output-Size should be 2 x 2. // Square output. - AdaptiveMeanPooling module1(2, 2); + AdaptiveMeanPooling<> module1(2, 2); module1.InputHeight() = 3; module1.InputWidth() = 4; module1.Forward(input, output); @@ -5187,7 +4478,7 @@ TEST_CASE("AdaptiveMeanPoolingTestCase", "[ANNLayerTest]") input(6) = 3; // Output-Size should be 1 x 2. // Rectangular output. - AdaptiveMeanPooling module2(1, 2); + AdaptiveMeanPooling<> module2(1, 2); module2.InputHeight() = 3; module2.InputWidth() = 3; module2.Forward(input, output); @@ -5209,7 +4500,7 @@ TEST_CASE("AdaptiveMeanPoolingTestCase", "[ANNLayerTest]") input(8) = 3; // Output-Size should be 3 x 3. // Square output. - AdaptiveMeanPooling module3(std::tuple(3, 3)); + AdaptiveMeanPooling<> module3(std::tuple(3, 3)); module3.InputHeight() = 4; module3.InputWidth() = 4; module3.Forward(input, output); @@ -5229,7 +4520,7 @@ TEST_CASE("AdaptiveMeanPoolingTestCase", "[ANNLayerTest]") input(4) = 3; // Output-Size should be 3 x 3. // Square output. - AdaptiveMeanPooling module4(std::tuple(3, 3)); + AdaptiveMeanPooling<> module4(std::tuple(3, 3)); module4.InputHeight() = 4; module4.InputWidth() = 6; module4.Forward(input, output); @@ -5241,226 +4532,224 @@ TEST_CASE("AdaptiveMeanPoolingTestCase", "[ANNLayerTest]") module4.Backward(input, output, delta); REQUIRE(arma::accu(delta) == 2.25); } -*/ -/* TEST_CASE("TransposedConvolutionalLayerOptionalParameterTest", "[ANNLayerTest]") { - Sequential* decoder = new Sequential(); + Sequential<>* decoder = new Sequential<>(); // Check if we can create an object without specifying output. - REQUIRE_NOTHROW(decoder->Add(24, 16, + REQUIRE_NOTHROW(decoder->Add>(24, 16, 5, 5, 1, 1, 0, 0, 10, 10)); - REQUIRE_NOTHROW(decoder->Add(16, 1, + REQUIRE_NOTHROW(decoder->Add>(16, 1, 15, 15, 1, 1, 1, 1, 14, 14)); delete decoder; } -*/ -// TEST_CASE("BatchNormWithMinBatchesTest", "[ANNLayerTest]") -// { -// arma::mat input, output, result, runningMean, runningVar, delta; +TEST_CASE("BatchNormWithMinBatchesTest", "[ANNLayerTest]") +{ + arma::mat input, output, result, runningMean, runningVar, delta; -// // The input test matrix is of the form 3 x 2 x 4 x 1 where -// // number of images are 3 and number of feature maps are 2. -// input = { { 1, 446, 42 }, -// { 2, 16, 63 }, -// { 3, 13, 63 }, -// { 4, 21, 21 }, -// { 1, 13, 11 }, -// { 32, 45, 42 }, -// { 22, 16, 63 }, -// { 32, 13, 42 } }; -// -// // Output calculated using torch.nn.BatchNorm2d(). -// result = { { -0.4786, 3.2634, -0.1338 }, -// { -0.4702, -0.3525, 0.0427 }, -// { -0.4618, -0.3777, 0.0427 }, -// { -0.4534, -0.3104, -0.3104 }, -// { -1.5429, -0.8486, -0.9643 }, -// { 0.2507, 1.0029, 0.8293 }, -// { -0.3279, -0.675, 2.0443 }, -// { 0.2507 , -0.8486 , 0.8293 } }; + // The input test matrix is of the form 3 x 2 x 4 x 1 where + // number of images are 3 and number of feature maps are 2. + input = { { 1, 446, 42 }, + { 2, 16, 63 }, + { 3, 13, 63 }, + { 4, 21, 21 }, + { 1, 13, 11 }, + { 32, 45, 42 }, + { 22, 16, 63 }, + { 32, 13, 42 } }; -// // Check correctness of batch normalization. -// BatchNorm<> module1(2, 1e-5, false, 0.1); -// module1.Reset(); -// module1.Forward(input, output); -// CheckMatrices(output, result, 1e-1); + // Output calculated using torch.nn.BatchNorm2d(). + result = { { -0.4786, 3.2634, -0.1338 }, + { -0.4702, -0.3525, 0.0427 }, + { -0.4618, -0.3777, 0.0427 }, + { -0.4534, -0.3104, -0.3104 }, + { -1.5429, -0.8486, -0.9643 }, + { 0.2507, 1.0029, 0.8293 }, + { -0.3279, -0.675, 2.0443 }, + { 0.2507 , -0.8486 , 0.8293 } }; -// // Check backward function. -// module1.Backward(input, output, delta); -// REQUIRE(arma::accu(delta) == Approx(0.0102676).epsilon(1e-5)); + // Check correctness of batch normalization. + BatchNorm<> module1(2, 1e-5, false, 0.1); + module1.Reset(); + module1.Forward(input, output); + CheckMatrices(output, result, 1e-1); -// // Check values for running mean and running variance. -// // Calculated using torch.nn.BatchNorm2d(). -// runningMean = arma::mat(2, 1); -// runningVar = arma::mat(2, 1); -// runningMean(0) = 5.7917; -// runningMean(1) = 2.76667; -// runningVar(0) = 1543.6545; -// runningVar(1) = 33.488; + // Check backward function. + module1.Backward(input, output, delta); + REQUIRE(arma::accu(delta) == Approx(0.0102676).epsilon(1e-5)); -// CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); -// CheckMatrices(runningVar, module1.TrainingVariance(), 1e-2); + // Check values for running mean and running variance. + // Calculated using torch.nn.BatchNorm2d(). + runningMean = arma::mat(2, 1); + runningVar = arma::mat(2, 1); + runningMean(0) = 5.7917; + runningMean(1) = 2.76667; + runningVar(0) = 1543.6545; + runningVar(1) = 33.488; -// // Check correctness of layer when running mean and variance -// // are updated using cumulative average. -// BatchNorm<> module2(2); -// module2.Reset(); -// module2.Forward(input, output); -// CheckMatrices(output, result, 1e-1); + CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); + CheckMatrices(runningVar, module1.TrainingVariance(), 1e-2); -// // Check values for running mean and running variance. -// // Calculated using torch.nn.BatchNorm2d(). -// runningMean(0) = 57.9167; -// runningMean(1) = 27.6667; -// runningVar(0) = 15427.5380; -// runningVar(1) = 325.8787; + // Check correctness of layer when running mean and variance + // are updated using cumulative average. + BatchNorm<> module2(2); + module2.Reset(); + module2.Forward(input, output); + CheckMatrices(output, result, 1e-1); -// CheckMatrices(runningMean, module2.TrainingMean(), 1e-2); -// CheckMatrices(runningVar, module2.TrainingVariance(), 1e-2); + // Check values for running mean and running variance. + // Calculated using torch.nn.BatchNorm2d(). + runningMean(0) = 57.9167; + runningMean(1) = 27.6667; + runningVar(0) = 15427.5380; + runningVar(1) = 325.8787; -// // Check correctness when model is testing. -// arma::mat deterministicOutput; -// module1.Deterministic() = true; -// module1.Forward(input, deterministicOutput); + CheckMatrices(runningMean, module2.TrainingMean(), 1e-2); + CheckMatrices(runningVar, module2.TrainingVariance(), 1e-2); -// result.clear(); -// result = { { -0.12195, 11.20426, 0.92158 }, -// { -0.0965, 0.259824, 1.4560 }, -// { -0.071054, 0.183567, 1.45607 }, -// { -0.045601, 0.3870852, 0.38708 }, -// { -0.305288, 1.7683, 1.4227 }, -// { 5.05166, 7.29812, 6.7797 }, -// { 3.323614, 2.2867, 10.4086 }, -// { 5.05166, 1.7683, 6.7797 } }; + // Check correctness when model is testing. + arma::mat deterministicOutput; + module1.Deterministic() = true; + module1.Forward(input, deterministicOutput); -// CheckMatrices(result, deterministicOutput, 1e-1); + result.clear(); + result = { { -0.12195, 11.20426, 0.92158 }, + { -0.0965, 0.259824, 1.4560 }, + { -0.071054, 0.183567, 1.45607 }, + { -0.045601, 0.3870852, 0.38708 }, + { -0.305288, 1.7683, 1.4227 }, + { 5.05166, 7.29812, 6.7797 }, + { 3.323614, 2.2867, 10.4086 }, + { 5.05166, 1.7683, 6.7797 } }; -// // Check correctness by updating the running mean and variance again. -// module1.Deterministic() = false; + CheckMatrices(result, deterministicOutput, 1e-1); -// // Clean up. -// output.clear(); -// input.clear(); + // Check correctness by updating the running mean and variance again. + module1.Deterministic() = false; -// // The input test matrix is of the form 2 x 2 x 3 x 1 where -// // number of images are 2 and number of feature maps are 2. -// input = { { 12, 443 }, -// { 134, 45 }, -// { 11, 13 }, -// { 14, 55 }, -// { 110, 4 }, -// { 1, 45 } }; -// -// result = { { -0.629337, 2.14791 }, -// { 0.156797, -0.416694 }, -// { -0.63578, -0.622893 }, -// { -0.637481, 0.4440386 }, -// { 1.894857, -0.901267 }, -// { -0.980402, 0.180253 } }; + // Clean up. + output.clear(); + input.clear(); -// module1.Forward(input, output); -// CheckMatrices(result, output, 1e-3); + // The input test matrix is of the form 2 x 2 x 3 x 1 where + // number of images are 2 and number of feature maps are 2. + input = { { 12, 443 }, + { 134, 45 }, + { 11, 13 }, + { 14, 55 }, + { 110, 4 }, + { 1, 45 } }; -// // Check correctness for the second module as well. -// module2.Forward(input, output); -// CheckMatrices(result, output, 1e-3); + result = { { -0.629337, 2.14791 }, + { 0.156797, -0.416694 }, + { -0.63578, -0.622893 }, + { -0.637481, 0.4440386 }, + { 1.894857, -0.901267 }, + { -0.980402, 0.180253 } }; -// // Calculated using torch.nn.BatchNorm2d(). -// runningMean(0) = 16.1792; -// runningMean(1) = 6.30667; -// runningVar(0) = 4276.5849; -// runningVar(1) = 202.595; + module1.Forward(input, output); + CheckMatrices(result, output, 1e-3); -// CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); -// CheckMatrices(runningVar, module1.TrainingVariance(), 1e-1); + // Check correctness for the second module as well. + module2.Forward(input, output); + CheckMatrices(result, output, 1e-3); -// // Check correctness of running mean and variance when their -// // values are updated using cumulative average. -// runningMean(0) = 83.79166; -// runningMean(1) = 32.9166; -// runningVar(0) = 22164.1035; -// runningVar(1) = 1025.2227; + // Calculated using torch.nn.BatchNorm2d(). + runningMean(0) = 16.1792; + runningMean(1) = 6.30667; + runningVar(0) = 4276.5849; + runningVar(1) = 202.595; -// CheckMatrices(runningMean, module2.TrainingMean(), 1e-3); -// CheckMatrices(runningVar, module2.TrainingVariance(), 1e-3); + CheckMatrices(runningMean, module1.TrainingMean(), 1e-3); + CheckMatrices(runningVar, module1.TrainingVariance(), 1e-1); -// // Check backward function. -// module1.Backward(input, output, delta); + // Check correctness of running mean and variance when their + // values are updated using cumulative average. + runningMean(0) = 83.79166; + runningMean(1) = 32.9166; + runningVar(0) = 22164.1035; + runningVar(1) = 1025.2227; -// deterministicOutput.clear(); -// module1.Deterministic() = true; -// module1.Forward(input, deterministicOutput); + CheckMatrices(runningMean, module2.TrainingMean(), 1e-3); + CheckMatrices(runningVar, module2.TrainingVariance(), 1e-3); -// result.clear(); -// result = { { -0.06388436, 6.524754114 }, -// { 1.799655281, 0.44047968 }, -// { -0.07913291, -0.04784981 }, -// { 0.5405045, 3.4210097 }, -// { 7.2851023, -0.1620577 }, -// { -0.37282639, 2.7184474 } }; + // Check backward function. + module1.Backward(input, output, delta); -// // Calculated using torch.nn.BatchNorm2d(). -// CheckMatrices(result, deterministicOutput, 1e-1); -// } + deterministicOutput.clear(); + module1.Deterministic() = true; + module1.Forward(input, deterministicOutput); -// /** -// * Batch Normalization layer numerical gradient test. -// */ -// TEST_CASE("GradientBatchNormWithMiniBatchesTest", "[ANNLayerTest]") -// { -// // Add function gradient instantiation. -// // To make this test robust, check it ten times. -// bool pass = false; -// for (size_t trial = 0; trial < 10; trial++) -// { -// struct GradientFunction -// { -// GradientFunction() : -// input(arma::randn(16, 1024)), -// target(arma::zeros(1, 1024)) -// { -// model = new FFN(); -// model->ResetData(input, target); -// model->Add>(); -// model->Add>(1, 2, 3, 3, 1, 1, 0, 0, 4, 4); -// model->Add>(2); -// model->Add>(2 * 2 * 2, 2); -// model->Add>(); -// } + result.clear(); + result = { { -0.06388436, 6.524754114 }, + { 1.799655281, 0.44047968 }, + { -0.07913291, -0.04784981 }, + { 0.5405045, 3.4210097 }, + { 7.2851023, -0.1620577 }, + { -0.37282639, 2.7184474 } }; -// ~GradientFunction() -// { -// delete model; -// } + // Calculated using torch.nn.BatchNorm2d(). + CheckMatrices(result, deterministicOutput, 1e-1); +} -// double Gradient(arma::mat& gradient) const -// { -// double error = model->Evaluate(model->Parameters(), 0, 1024, false); -// model->Gradient(model->Parameters(), 0, gradient, 1024); -// return error; -// } +/** + * Batch Normalization layer numerical gradient test. + */ +TEST_CASE("GradientBatchNormWithMiniBatchesTest", "[ANNLayerTest]") +{ + // Add function gradient instantiation. + // To make this test robust, check it ten times. + bool pass = false; + for (size_t trial = 0; trial < 10; trial++) + { + struct GradientFunction + { + GradientFunction() : + input(arma::randn(16, 1024)), + target(arma::zeros(1, 1024)) + { + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add>(); + model->Add>(1, 2, 3, 3, 1, 1, 0, 0, 4, 4); + model->Add>(2); + model->Add>(2 * 2 * 2, 2); + model->Add>(); + } -// arma::mat& Parameters() { return model->Parameters(); } + ~GradientFunction() + { + delete model; + } -// FFN* model; -// arma::mat input, target; -// } function; + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1024, false); + model->Gradient(model->Parameters(), 0, gradient, 1024); + return error; + } -// double gradient = CheckGradient(function); -// if (gradient < 1e-1) -// { -// pass = true; -// break; -// } -// } + arma::mat& Parameters() { return model->Parameters(); } -// REQUIRE(pass); -// } + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + double gradient = CheckGradient(function); + if (gradient < 1e-1) + { + pass = true; + break; + } + } + + REQUIRE(pass); +} TEST_CASE("ConvolutionLayerTestCase", "[ANNLayerTest]") { @@ -5477,116 +4766,81 @@ TEST_CASE("ConvolutionLayerTestCase", "[ANNLayerTest]") { 22, 16 , 63 }, { 32, 13 , 42 } }; - Convolution layer(4, 1, 1, 1, 1, 0, 0); - layer.InputDimensions() = std::vector({ 4, 1, 2 }); - layer.ComputeOutputDimensions(); - arma::mat layerWeights(layer.WeightSize(), 1); - layer.SetWeights(layerWeights.memptr()); - output.set_size(layer.OutputSize(), 3); + Convolution<> layer(2, 4, 1, 1, 1, 1, 0, 0, 4, 1); + layer.Reset(); // Set weights to 1.0 and bias to 0.0. - layer.Weight().fill(1.0); - layer.Bias().zeros(); + layer.Parameters().zeros(); + arma::mat weight(2 * 4, 1); + weight.fill(1.0); + layer.Parameters().submat(arma::span(0, 2 * 4 - 1), arma::span()) = weight; layer.Forward(input, output); // Value calculated using torch.nn.Conv2d(). REQUIRE(arma::accu(output) == 4108); // Set bias to one. - layer.Bias().fill(1.0); + layer.Parameters().fill(1.0); layer.Forward(input, output); // Value calculated using torch.nn.Conv2d(). REQUIRE(arma::accu(output) == 4156); } -// TEST_CASE("BatchNormDeterministicTest", "[ANNLayerTest]") -// { -// FFN<> module; -// module.Add>(2, 1e-5, false); -// module.Add>(); +TEST_CASE("BatchNormDeterministicTest", "[ANNLayerTest]") +{ + FFN<> module; + module.Add>(2, 1e-5, false); + module.Add>(); -// arma::mat input(4, 3), output; -// module.ResetParameters(); + arma::mat input(4, 3), output; + module.ResetParameters(); -// // The model should switch to Deterministic mode for predicting. -// module.Predict(input, output); -// REQUIRE(boost::get*>(module.Model()[0])->Deterministic() == true); + // The model should switch to Deterministic mode for predicting. + module.Predict(input, output); + REQUIRE(boost::get*>(module.Model()[0])->Deterministic() == true); -// output.ones(); -// module.Train(input, output); -// // The model should switch to training mode for predicting. -// REQUIRE(boost::get*>(module.Model()[0])->Deterministic() == 0); -// } - -// /** -// * Linear module weight initialization test. -// */ -// TEST_CASE("LinearLayerWeightInitializationTest", "[ANNLayerTest]") -// { -// size_t inSize = 10, outSize = 4; -// Linear<> linear = Linear<>(inSize, outSize); -// linear.Reset(); -// RandomInitialization().Initialize(linear.Weight()); -// linear.Bias().ones(); - -// REQUIRE(std::equal(linear.Weight().begin(), -// linear.Weight().end(), linear.Parameters().begin())); - -// REQUIRE(std::equal(linear.Bias().begin(), -// linear.Bias().end(), linear.Parameters().begin() + inSize * outSize)); - -// REQUIRE(linear.Weight().n_rows == outSize); -// REQUIRE(linear.Weight().n_cols == inSize); -// REQUIRE(linear.Bias().n_rows == outSize); -// REQUIRE(linear.Bias().n_cols == 1); -// REQUIRE(linear.Parameters().n_rows == inSize * outSize + outSize); -// } - -// /** -// * Atrous Convolution module weight initialization test. -// */ -// TEST_CASE("AtrousConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") -// { -// size_t inSize = 2, outSize = 3; -// size_t kernelWidth = 4, kernelHeight = 5; -// AtrousConvolution<> module = AtrousConvolution<>(inSize, outSize, -// kernelWidth, kernelHeight, 6, 7, std::make_tuple(8, 9), -// std::make_tuple(10, 11), 12, 13, 14, 15); -// module.Reset(); -// RandomInitialization().Initialize(module.Weight()); -// module.Bias().ones(); - -// REQUIRE(std::equal(module.Weight().begin(), -// module.Weight().end(), module.Parameters().begin())); - -// REQUIRE(std::equal(module.Bias().begin(), -// module.Bias().end(), module.Parameters().end() - outSize)); - -// REQUIRE(module.Weight().n_rows == kernelWidth); -// REQUIRE(module.Weight().n_cols == kernelHeight); -// REQUIRE(module.Weight().n_slices == inSize * outSize); -// REQUIRE(module.Bias().n_rows == outSize); -// REQUIRE(module.Bias().n_cols == 1); -// REQUIRE(module.Parameters().n_rows -// == (outSize * inSize * kernelWidth * kernelHeight) + outSize); -// } + output.ones(); + module.Train(input, output); + // The model should switch to training mode for predicting. + REQUIRE(boost::get*>(module.Model()[0])->Deterministic() == 0); +} /** - * Convolution module weight initialization test. + * Linear module weight initialization test. */ -TEST_CASE("ConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") +TEST_CASE("LinearLayerWeightInitializationTest", "[ANNLayerTest]") +{ + size_t inSize = 10, outSize = 4; + Linear<> linear = Linear<>(inSize, outSize); + linear.Reset(); + RandomInitialization().Initialize(linear.Weight()); + linear.Bias().ones(); + + REQUIRE(std::equal(linear.Weight().begin(), + linear.Weight().end(), linear.Parameters().begin())); + + REQUIRE(std::equal(linear.Bias().begin(), + linear.Bias().end(), linear.Parameters().begin() + inSize * outSize)); + + REQUIRE(linear.Weight().n_rows == outSize); + REQUIRE(linear.Weight().n_cols == inSize); + REQUIRE(linear.Bias().n_rows == outSize); + REQUIRE(linear.Bias().n_cols == 1); + REQUIRE(linear.Parameters().n_rows == inSize * outSize + outSize); +} + +/** + * Atrous Convolution module weight initialization test. + */ +TEST_CASE("AtrousConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") { size_t inSize = 2, outSize = 3; size_t kernelWidth = 4, kernelHeight = 5; - Convolution module = Convolution(outSize, - kernelWidth, kernelHeight, 6, 7, std::tuple(8, 9), - std::tuple(10, 11), "none"); - module.InputDimensions() = std::vector({ 12, 13, 2 }); - module.ComputeOutputDimensions(); - arma::mat weights(module.WeightSize(), 1); - module.SetWeights(weights.memptr()); - + AtrousConvolution<> module = AtrousConvolution<>(inSize, outSize, + kernelWidth, kernelHeight, 6, 7, std::make_tuple(8, 9), + std::make_tuple(10, 11), 12, 13, 14, 15); + module.Reset(); RandomInitialization().Initialize(module.Weight()); module.Bias().ones(); @@ -5598,7 +4852,36 @@ TEST_CASE("ConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") REQUIRE(module.Weight().n_rows == kernelWidth); REQUIRE(module.Weight().n_cols == kernelHeight); - REQUIRE(module.Weight().n_slices == outSize * inSize); + REQUIRE(module.Weight().n_slices == inSize * outSize); + REQUIRE(module.Bias().n_rows == outSize); + REQUIRE(module.Bias().n_cols == 1); + REQUIRE(module.Parameters().n_rows + == (outSize * inSize * kernelWidth * kernelHeight) + outSize); +} + +/** + * Convolution module weight initialization test. + */ +TEST_CASE("ConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") +{ + size_t inSize = 2, outSize = 3; + size_t kernelWidth = 4, kernelHeight = 5; + Convolution<> module = Convolution<>(inSize, outSize, + kernelWidth, kernelHeight, 6, 7, std::tuple(8, 9), + std::tuple(10, 11), 12, 13, "none"); + module.Reset(); + RandomInitialization().Initialize(module.Weight()); + module.Bias().ones(); + + REQUIRE(std::equal(module.Weight().begin(), + module.Weight().end(), module.Parameters().begin())); + + REQUIRE(std::equal(module.Bias().begin(), + module.Bias().end(), module.Parameters().end() - outSize)); + + REQUIRE(module.Weight().n_rows == kernelWidth); + REQUIRE(module.Weight().n_cols == kernelHeight); + REQUIRE(module.Weight().n_slices == inSize * outSize); REQUIRE(module.Bias().n_rows == outSize); REQUIRE(module.Bias().n_cols == 1); REQUIRE(module.Parameters().n_rows @@ -5607,12 +4890,12 @@ TEST_CASE("ConvolutionLayerWeightInitializationTest", "[ANNLayerTest]") /** * Transposed Convolution module weight initialization test. - * + */ TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]") { size_t inSize = 3, outSize = 3; size_t kernelWidth = 4, kernelHeight = 4; - TransposedConvolution module = TransposedConvolution(inSize, outSize, + TransposedConvolution<> module = TransposedConvolution<>(inSize, outSize, kernelWidth, kernelHeight, 1, 1, 1, 1, 5, 5, 6, 6); module.Reset(); RandomInitialization().Initialize(module.Weight()); @@ -5632,263 +4915,262 @@ TEST_CASE("TransposedConvolutionWeightInitializationTest", "[ANNLayerTest]") REQUIRE(module.Parameters().n_rows == (outSize * inSize * kernelWidth * kernelHeight) + outSize); } -*/ /** * Simple Test for ChannelShuffle layer. */ -// TEST_CASE("ChannelShuffleLayerTest", "[ANNLayerTest]") -// { -// arma::mat input1, output1, outputExpected1, outputBackward1; -// ChannelShuffle<> module1(2, 2, 6, 2); -// -// input1 << 1 << 13 << arma::endr -// << 2 << 14 << arma::endr -// << 3 << 15 << arma::endr -// << 4 << 16 << arma::endr -// << 5 << 17 << arma::endr -// << 6 << 18 << arma::endr -// << 7 << 19 << arma::endr -// << 8 << 20 << arma::endr -// << 9 << 21 << arma::endr -// << 10 << 22 << arma::endr -// << 11 << 23 << arma::endr -// << 12 << 24 << arma::endr; -// input1.reshape(24, 1); -// // Value calculated using torch.nn.ChannelShuffle(). -// outputExpected1 << 1 << 17 << arma::endr -// << 2 << 18 << arma::endr -// << 3 << 19 << arma::endr -// << 4 << 20 << arma::endr -// << 13 << 9 << arma::endr -// << 14 << 10 << arma::endr -// << 15 << 11 << arma::endr -// << 16 << 12 << arma::endr -// << 5 << 21 << arma::endr -// << 6 << 22 << arma::endr -// << 7 << 23 << arma::endr -// << 8 << 24 << arma::endr; -// outputExpected1.reshape(24, 1); -// // Check the Forward pass of the layer. -// module1.Forward(input1, output1); -// CheckMatrices(output1, outputExpected1); -// -// // Check the Backward pass of the layer. -// module1.Backward(output1, output1, outputBackward1); -// CheckMatrices(input1, outputBackward1); -// -// } +TEST_CASE("ChannelShuffleLayerTest", "[ANNLayerTest]") +{ + arma::mat input1, output1, outputExpected1, outputBackward1; + ChannelShuffle<> module1(2, 2, 6, 2); + + input1 << 1 << 13 << arma::endr + << 2 << 14 << arma::endr + << 3 << 15 << arma::endr + << 4 << 16 << arma::endr + << 5 << 17 << arma::endr + << 6 << 18 << arma::endr + << 7 << 19 << arma::endr + << 8 << 20 << arma::endr + << 9 << 21 << arma::endr + << 10 << 22 << arma::endr + << 11 << 23 << arma::endr + << 12 << 24 << arma::endr; + input1.reshape(24, 1); + // Value calculated using torch.nn.ChannelShuffle(). + outputExpected1 << 1 << 17 << arma::endr + << 2 << 18 << arma::endr + << 3 << 19 << arma::endr + << 4 << 20 << arma::endr + << 13 << 9 << arma::endr + << 14 << 10 << arma::endr + << 15 << 11 << arma::endr + << 16 << 12 << arma::endr + << 5 << 21 << arma::endr + << 6 << 22 << arma::endr + << 7 << 23 << arma::endr + << 8 << 24 << arma::endr; + outputExpected1.reshape(24, 1); + // Check the Forward pass of the layer. + module1.Forward(input1, output1); + CheckMatrices(output1, outputExpected1); + + // Check the Backward pass of the layer. + module1.Backward(output1, output1, outputBackward1); + CheckMatrices(input1, outputBackward1); + +} /** * Simple Test for PixelShuffle layer. */ -// TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]") -// { -// arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1; -// arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2; -// PixelShuffle<> module1(2, 2, 2, 4); -// PixelShuffle<> module2(2, 2, 2, 4); -// -// // Input is a single image, of size (2,2) and having 4 channels. -// input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << arma::endr; -// gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 -// << 12 << 16 << arma::endr; -// -// // Calculated using torch.nn.PixelShuffle(). -// outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 -// << 0 << 0 << 0 << 0 << arma::endr; -// gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 -// << 6 << 14 << 8 << 16 << arma::endr; -// -// input1 = input1.t(); -// outputExpected1 = outputExpected1.t(); -// gy1 = gy1.t(); -// gExpected1 = gExpected1.t(); -// -// // Check the Forward pass of the layer. -// module1.Forward(input1, output1); -// CheckMatrices(output1, outputExpected1); -// -// // Check the Backward pass of the layer. -// module1.Backward(input1, gy1, g1); -// CheckMatrices(g1, gExpected1); -// -// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. -// input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; -// gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 -// << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30 -// << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr; -// -// // Calculated using torch.nn.PixelShuffle(). -// outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 -// << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0 -// << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr; -// gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 -// << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29 -// << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr; -// -// input2 = input2.t(); -// outputExpected2 = outputExpected2.t(); -// gy2 = gy2.t(); -// gExpected2 = gExpected2.t(); -// -// // Check the Forward pass of the layer. -// module2.Forward(input2, output2); -// CheckMatrices(output2, outputExpected2); -// -// // Check the Backward pass of the layer. -// module2.Backward(input2, gy2, g2); -// CheckMatrices(g2, gExpected2); -// } +TEST_CASE("PixelShuffleLayerTest", "[ANNLayerTest]") +{ + arma::mat input1, output1, gy1, g1, outputExpected1, gExpected1; + arma::mat input2, output2, gy2, g2, outputExpected2, gExpected2; + PixelShuffle<> module1(2, 2, 2, 4); + PixelShuffle<> module2(2, 2, 2, 4); + + // Input is a single image, of size (2,2) and having 4 channels. + input1 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << arma::endr; + gy1 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 + << 12 << 16 << arma::endr; + + // Calculated using torch.nn.PixelShuffle(). + outputExpected1 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 + << 0 << 0 << 0 << 0 << arma::endr; + gExpected1 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 + << 6 << 14 << 8 << 16 << arma::endr; + + input1 = input1.t(); + outputExpected1 = outputExpected1.t(); + gy1 = gy1.t(); + gExpected1 = gExpected1.t(); + + // Check the Forward pass of the layer. + module1.Forward(input1, output1); + CheckMatrices(output1, outputExpected1); + + // Check the Backward pass of the layer. + module1.Backward(input1, gy1, g1); + CheckMatrices(g1, gExpected1); + + // Input is a batch of 2 images, each of size (2,2) and having 4 channels. + input2 << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; + gy2 << 1 << 5 << 9 << 13 << 2 << 6 << 10 << 14 << 3 << 7 << 11 << 15 << 4 << 8 + << 12 << 16 << arma::endr << 17 << 21 << 25 << 29 << 18 << 22 << 26 << 30 + << 19 << 23 << 27 << 31 << 20 << 24 << 28 << 32 << arma::endr; + + // Calculated using torch.nn.PixelShuffle(). + outputExpected2 << 1 << 0 << 3 << 0 << 0 << 0 << 0 << 0 << 2 << 0 << 4 << 0 + << 0 << 0 << 0 << 0 << arma::endr << 5 << 0 << 7 << 0 << 0 << 0 << 0 << 0 + << 6 << 0 << 8 << 0 << 0 << 0 << 0 << 0 << arma::endr; + gExpected2 << 1 << 9 << 3 << 11 << 5 << 13 << 7 << 15 << 2 << 10 << 4 << 12 + << 6 << 14 << 8 << 16 << arma::endr << 17 << 25 << 19 << 27 << 21 << 29 + << 23 << 31 << 18 << 26 << 20 << 28 << 22 << 30 << 24 << 32 << arma::endr; + + input2 = input2.t(); + outputExpected2 = outputExpected2.t(); + gy2 = gy2.t(); + gExpected2 = gExpected2.t(); + + // Check the Forward pass of the layer. + module2.Forward(input2, output2); + CheckMatrices(output2, outputExpected2); + + // Check the Backward pass of the layer. + module2.Backward(input2, gy2, g2); + CheckMatrices(g2, gExpected2); +} /** * Test that the function that can access the parameters of the * PixelShuffle layer works. */ -// TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]") -// { -// // Create the layer using the empty constructor. -// PixelShuffle<> layer; -// -// // Set the different input parameters of the layer. -// layer.UpscaleFactor() = 2; -// layer.InputHeight() = 2; -// layer.InputWidth() = 2; -// layer.InputChannels() = 4; -// -// // Make sure we can get the parameters successfully. -// REQUIRE(layer.UpscaleFactor() == 2); -// REQUIRE(layer.InputHeight() == 2); -// REQUIRE(layer.InputWidth() == 2); -// REQUIRE(layer.InputChannels() == 4); -// -// arma::mat input, output; -// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. -// input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 -// << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; -// input = input.t(); -// layer.Forward(input, output); -// -// // Check whether output parameters are returned correctly. -// REQUIRE(layer.OutputHeight() == 4); -// REQUIRE(layer.OutputWidth() == 4); -// REQUIRE(layer.OutputChannels() == 1); -// } +TEST_CASE("PixelShuffleLayerParametersTest", "[ANNLayerTest]") +{ + // Create the layer using the empty constructor. + PixelShuffle<> layer; -// /** -// * Simple Test for SpatialDropout layer. -// */ -// TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]") -// { -// arma::mat input, output, gy, g, temp; -// arma::mat outputsExpected = arma::zeros(8, 12); -// arma::mat gsExpected = arma::zeros(8, 12); + // Set the different input parameters of the layer. + layer.UpscaleFactor() = 2; + layer.InputHeight() = 2; + layer.InputWidth() = 2; + layer.InputChannels() = 4; -// // Set the seed to a random value. -// arma::arma_rng::set_seed_random(); -// SpatialDropout<> module(3, 0.2); + // Make sure we can get the parameters successfully. + REQUIRE(layer.UpscaleFactor() == 2); + REQUIRE(layer.InputHeight() == 2); + REQUIRE(layer.InputWidth() == 2); + REQUIRE(layer.InputChannels() == 4); -// // Input is a batch of 2 images, each of size (2,2) and having 4 channels. -// input = { 0.4963, 0.0885, 0.7682, 0.1320, 0.3074, 0.4901, 0.6341, 0.8964, -// 0.4556, 0.3489, 0.6323, 0.4017 }; -// -// gy = { 1, 3, 2, 4, 5, 7, 6, 8, 9, 11, 10, 12 }; -// -// // Following values have been calculated using torch.nn.Dropout2d(p=0.2). -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -// outputsExpected.row(0) = temp; -// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0.5695, 0.4361, 0.7904, -// 0.5021 }; -// outputsExpected.row(1) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0.5695, 0.4361, -// 0.7904, 0.5021 }; -// outputsExpected.row(2) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, 0, -// 0, 0, 0 }; -// outputsExpected.row(3) = temp; -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0.5695, 0.4361, 0.7904, 0.5021 }; -// outputsExpected.row(4) = temp; -// temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0, 0, 0, 0 }; -// outputsExpected.row(5) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0, 0, 0, 0 }; -// outputsExpected.row(6) = temp; -// temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, -// 0.5695, 0.4361, 0.7904, 0.5021 }; -// outputsExpected.row(7) = temp; -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -// gsExpected.row(0) = temp; -// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 11.2500, 13.7500, -// 12.5000, 15.0000 }; -// gsExpected.row(1) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 11.2500, 13.7500, -// 12.5000, 15.0000 }; -// gsExpected.row(2) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, 0, -// 0, 0, 0 }; -// gsExpected.row(3) = temp; -// temp = { 0, 0, 0, 0, 0, 0, 0, 0, 11.2500, 13.7500, 12.5000, 15.0000 }; -// gsExpected.row(4) = temp; -// temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 0, 0, 0, 0 }; -// gsExpected.row(5) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 0, 0, 0, 0 }; -// gsExpected.row(6) = temp; -// temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, -// 11.2500, 13.7500, 12.5000, 15.0000 }; -// gsExpected.row(7) = temp; + arma::mat input, output; + // Input is a batch of 2 images, each of size (2,2) and having 4 channels. + input << 1 << 3 << 2 << 4 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << arma::endr << 5 << 7 << 6 << 8 << 0 << 0 << 0 << 0 << 0 << 0 + << 0 << 0 << 0 << 0 << 0 << 0 << arma::endr; + input = input.t(); + layer.Forward(input, output); -// input = input.t(); -// gy = gy.t(); -// outputsExpected = outputsExpected.t(); -// gsExpected = gsExpected.t(); + // Check whether output parameters are returned correctly. + REQUIRE(layer.OutputHeight() == 4); + REQUIRE(layer.OutputWidth() == 4); + REQUIRE(layer.OutputChannels() == 1); +} -// // Compute the Forward and Backward passes and store the results. -// module.Forward(input, output); -// module.Backward(input, gy, g); +/* + * Simple Test for SpatialDropout layer. + */ +TEST_CASE("SpatialDropoutLayerTest", "[ANNLayerTest]") +{ + arma::mat input, output, gy, g, temp; + arma::mat outputsExpected = arma::zeros(8, 12); + arma::mat gsExpected = arma::zeros(8, 12); -// // Check through all possible cases, to find a match and then compare results. -// for (size_t i = 0; i < outputsExpected.n_cols; ++i) -// { -// if (arma::approx_equal(outputsExpected.col(i), output, "absdiff", 1e-1)) -// { -// // Check the correctness of the Forward pass of the layer. -// CheckMatrices(output, outputsExpected.col(i), 1e-1); -// // Check the correctness of the Backward pass of the layer. -// CheckMatrices(g, gsExpected.col(i), 1e-1); -// } -// } + // Set the seed to a random value. + arma::arma_rng::set_seed_random(); + SpatialDropout<> module(3, 0.2); -// // Check if the output is same as input when using deterministic mode. -// module.Deterministic() = true; -// output.clear(); -// module.Forward(input, output); -// CheckMatrices(output, input, 1e-1); -// } + // Input is a batch of 2 images, each of size (2,2) and having 4 channels. + input = { 0.4963, 0.0885, 0.7682, 0.1320, 0.3074, 0.4901, 0.6341, 0.8964, + 0.4556, 0.3489, 0.6323, 0.4017 }; -// /** -// * Test that the function that can access the parameters of the -// * SpatialDropout layer works. -// */ -// TEST_CASE("SpatialDropoutLayerParametersTest", "[ANNLayerTest]") -// { -// // Create the layer using the empty constructor. -// SpatialDropout<> layer; + gy = { 1, 3, 2, 4, 5, 7, 6, 8, 9, 11, 10, 12 }; -// // Set the input parameters. -// layer.Size() = 3; -// layer.Ratio(0.2); + // Following values have been calculated using torch.nn.Dropout2d(p=0.2). + temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + outputsExpected.row(0) = temp; + temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0.5695, 0.4361, 0.7904, + 0.5021 }; + outputsExpected.row(1) = temp; + temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0.5695, 0.4361, + 0.7904, 0.5021 }; + outputsExpected.row(2) = temp; + temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, 0, + 0, 0, 0 }; + outputsExpected.row(3) = temp; + temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0.5695, 0.4361, 0.7904, 0.5021 }; + outputsExpected.row(4) = temp; + temp = { 0, 0, 0, 0, 0.3842, 0.6126, 0.7926, 1.1205, 0, 0, 0, 0 }; + outputsExpected.row(5) = temp; + temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0, 0, 0, 0, 0, 0, 0, 0 }; + outputsExpected.row(6) = temp; + temp = { 0.6204, 0.1106, 0.9603, 0.1650, 0.3842, 0.6126, 0.7926, 1.1205, + 0.5695, 0.4361, 0.7904, 0.5021 }; + outputsExpected.row(7) = temp; + temp = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + gsExpected.row(0) = temp; + temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 11.2500, 13.7500, + 12.5000, 15.0000 }; + gsExpected.row(1) = temp; + temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 11.2500, 13.7500, + 12.5000, 15.0000 }; + gsExpected.row(2) = temp; + temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, 0, + 0, 0, 0 }; + gsExpected.row(3) = temp; + temp = { 0, 0, 0, 0, 0, 0, 0, 0, 11.2500, 13.7500, 12.5000, 15.0000 }; + gsExpected.row(4) = temp; + temp = { 0, 0, 0, 0, 6.2500, 8.7500, 7.5000, 10.0000, 0, 0, 0, 0 }; + gsExpected.row(5) = temp; + temp = { 1.2500, 3.7500, 2.5000, 5.0000, 0, 0, 0, 0, 0, 0, 0, 0 }; + gsExpected.row(6) = temp; + temp = { 1.2500, 3.7500, 2.5000, 5.0000, 6.2500, 8.7500, 7.5000, 10.0000, + 11.2500, 13.7500, 12.5000, 15.0000 }; + gsExpected.row(7) = temp; -// // Check whether the input parameters have been set correctly. -// REQUIRE(layer.Size() == 3); -// REQUIRE(layer.Ratio() == 0.2); -// } + input = input.t(); + gy = gy.t(); + outputsExpected = outputsExpected.t(); + gsExpected = gsExpected.t(); + + // Compute the Forward and Backward passes and store the results. + module.Forward(input, output); + module.Backward(input, gy, g); + + // Check through all possible cases, to find a match and then compare results. + for (size_t i = 0; i < outputsExpected.n_cols; ++i) + { + if (arma::approx_equal(outputsExpected.col(i), output, "absdiff", 1e-1)) + { + // Check the correctness of the Forward pass of the layer. + CheckMatrices(output, outputsExpected.col(i), 1e-1); + // Check the correctness of the Backward pass of the layer. + CheckMatrices(g, gsExpected.col(i), 1e-1); + } + } + + // Check if the output is same as input when using deterministic mode. + module.Deterministic() = true; + output.clear(); + module.Forward(input, output); + CheckMatrices(output, input, 1e-1); +} + +/** + * Test that the function that can access the parameters of the + * SpatialDropout layer works. + */ +TEST_CASE("SpatialDropoutLayerParametersTest", "[ANNLayerTest]") +{ + // Create the layer using the empty constructor. + SpatialDropout<> layer; + + // Set the input parameters. + layer.Size() = 3; + layer.Ratio(0.2); + + // Check whether the input parameters have been set correctly. + REQUIRE(layer.Size() == 3); + REQUIRE(layer.Ratio() == 0.2); +} /** * Simple Positional Encoding layer test. - * + */ TEST_CASE("SimplePositionalEncodingTest", "[ANNLayerTest]") { const size_t seqLength = 5; @@ -5899,7 +5181,7 @@ TEST_CASE("SimplePositionalEncodingTest", "[ANNLayerTest]") arma::mat gy = 0.01 * arma::randu(embedDim * seqLength, batchSize); arma::mat output, g; - PositionalEncoding module(embedDim, seqLength); + PositionalEncoding<> module(embedDim, seqLength); // Check Forward function. module.Forward(input, output); @@ -5910,11 +5192,10 @@ TEST_CASE("SimplePositionalEncodingTest", "[ANNLayerTest]") module.Backward(input, gy, g); REQUIRE(std::equal(gy.begin(), gy.end(), g.begin())); } -*/ /** * Jacobian test for Positional Encoding layer. - * + */ TEST_CASE("JacobianPositionalEncodingTest", "[ANNLayerTest]") { for (size_t i = 0; i < 5; ++i) @@ -5924,17 +5205,16 @@ TEST_CASE("JacobianPositionalEncodingTest", "[ANNLayerTest]") arma::mat input; input.set_size(embedDim * seqLength, 1); - PositionalEncoding module(embedDim, seqLength); + PositionalEncoding<> module(embedDim, seqLength); double error = JacobianTest(module, input); REQUIRE(error <= 1e-5); } } -*/ /** * Simple Multihead Attention test. - * + */ TEST_CASE("SimpleMultiheadAttentionTest", "[ANNLayerTest]") { size_t tLen = 5; @@ -5959,7 +5239,7 @@ TEST_CASE("SimpleMultiheadAttentionTest", "[ANNLayerTest]") arma::mat keyPaddingMask = arma::zeros(1, sLen); keyPaddingMask(sLen - 1) = std::numeric_limits::lowest(); - MultiheadAttention module(tLen, sLen, embedDim, numHeads); + MultiheadAttention<> module(tLen, sLen, embedDim, numHeads); module.AttentionMask() = attnMask; module.KeyPaddingMask() = keyPaddingMask; module.Reset(); @@ -5986,11 +5266,10 @@ TEST_CASE("SimpleMultiheadAttentionTest", "[ANNLayerTest]") REQUIRE(gradient.n_rows == module.Parameters().n_rows); REQUIRE(gradient.n_cols == module.Parameters().n_cols); } -*/ /** * Jacobian MultiheadAttention module test. - * + */ TEST_CASE("JacobianMultiheadAttentionTest", "[ANNLayerTest]") { // Check when query = key = value. @@ -6004,7 +5283,7 @@ TEST_CASE("JacobianMultiheadAttentionTest", "[ANNLayerTest]") arma::mat query = arma::randu(embedDim * tgtSeqLen, batchSize); arma::mat input = arma::join_cols(arma::join_cols(query, query), query); - MultiheadAttention module(tgtSeqLen, tgtSeqLen, embedDim, nHeads); + MultiheadAttention<> module(tgtSeqLen, tgtSeqLen, embedDim, nHeads); module.Parameters().randu(); double error = CustomJacobianTest(module, input); @@ -6024,7 +5303,7 @@ TEST_CASE("JacobianMultiheadAttentionTest", "[ANNLayerTest]") arma::mat key = 0.091 * arma::randu(embedDim * srcSeqLen, batchSize); arma::mat input = arma::join_cols(arma::join_cols(query, key), key); - MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); + MultiheadAttention<> module(tgtSeqLen, srcSeqLen, embedDim, nHeads); module.Parameters().randu(); double error = CustomJacobianTest(module, input); @@ -6045,18 +5324,17 @@ TEST_CASE("JacobianMultiheadAttentionTest", "[ANNLayerTest]") arma::mat value = 0.045 * arma::randu(embedDim * srcSeqLen, batchSize); arma::mat input = arma::join_cols(arma::join_cols(query, key), value); - MultiheadAttention module(tgtSeqLen, srcSeqLen, embedDim, nHeads); + MultiheadAttention<> module(tgtSeqLen, srcSeqLen, embedDim, nHeads); module.Parameters().randu(); double error = JacobianTest(module, input); REQUIRE(error <= 1e-5); } } -*/ /** * Numerical gradient test for MultiheadAttention layer. - * + */ TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]") { struct GradientFunction @@ -6090,17 +5368,16 @@ TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]") keyPaddingMask = arma::zeros(1, srcSeqLen); keyPaddingMask(srcSeqLen - 1) = std::numeric_limits::lowest(); - model = new FFN(); - model->ResetData(input, target); - // attnModule = new MultiheadAttention(tgtSeqLen, srcSeqLen, embedDim, - // nHeads); - // attnModule->AttentionMask() = attnMask; - // attnModule->KeyPaddingMask() = keyPaddingMask; - // model->Add(attnModule); - model->Add(tgtSeqLen, srcSeqLen, embedDim, nHeads, - attnMask, keyPaddingMask); - model->Add(embedDim * tgtSeqLen, vocabSize); - model->Add(); + model = new FFN, XavierInitialization>(); + model->Predictors() = input; + model->Responses() = target; + attnModule = new MultiheadAttention<>(tgtSeqLen, srcSeqLen, + embedDim, nHeads); + attnModule->AttentionMask() = attnMask; + attnModule->KeyPaddingMask() = keyPaddingMask; + model->Add(attnModule); + model->Add>(embedDim * tgtSeqLen, vocabSize); + model->Add>(); } ~GradientFunction() @@ -6117,8 +5394,8 @@ TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; - // MultiheadAttention* attnModule; + FFN, XavierInitialization>* model; + MultiheadAttention<>* attnModule; arma::mat input, target, attnMask, keyPaddingMask; const size_t tgtSeqLen; @@ -6131,11 +5408,10 @@ TEST_CASE("GradientMultiheadAttentionTest", "[ANNLayerTest]") REQUIRE(CheckGradient(function) <= 3e-06); } -*/ /** * Simple tests for instance normalization layer. - * + */ TEST_CASE("InstanceNormLayerTest", "[ANNLayerTest]") { arma::mat input, result, output, delta, deltaExpected; @@ -6250,12 +5526,11 @@ TEST_CASE("InstanceNormLayerTest", "[ANNLayerTest]") CheckMatrices(output, result, 1e-1); } -*/ /** * Test that the functions that can access the parameters of the * Instance Norm layer work. - * + */ TEST_CASE("InstanceNormLayerParametersTest", "[ANNLayerTest]") { // Parameter order : size, eps. @@ -6273,11 +5548,10 @@ TEST_CASE("InstanceNormLayerParametersTest", "[ANNLayerTest]") CheckMatrices(layer.TrainingVariance(), runningVariance); CheckMatrices(layer.TrainingMean(), runningMean); } -*/ /** * Instance Norm layer numerical gradient test. - * + */ TEST_CASE("GradientInstanceNormLayerTest", "[ANNLayerTest]") { // Add function gradient instantiation. @@ -6293,8 +5567,9 @@ TEST_CASE("GradientInstanceNormLayerTest", "[ANNLayerTest]") arma::mat target; target.ones(1, 1024); - model = new FFN(); - model->ResetData(input, target); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; model->Add >(); model->Add >(1, 2, 3, 3, 1, 1, 0, 0, 4, 4); model->Add > (2, 1024); @@ -6316,7 +5591,7 @@ TEST_CASE("GradientInstanceNormLayerTest", "[ANNLayerTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, NguyenWidrowInitialization>* model; arma::mat input, target; } function; @@ -6330,4 +5605,3 @@ TEST_CASE("GradientInstanceNormLayerTest", "[ANNLayerTest]") REQUIRE(pass); } -*/ diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann_test_tools.hpp index d9f485484e..dff9bbf5a9 100644 --- a/src/mlpack/tests/ann_test_tools.hpp +++ b/src/mlpack/tests/ann_test_tools.hpp @@ -17,24 +17,41 @@ using namespace mlpack; using namespace mlpack::ann; +// Helper function which calls the Reset function of the given module. +template +void ResetFunction( + T& layer, + typename std::enable_if::value>::type* = 0) +{ + layer.Reset(); +} + +template +void ResetFunction( + T& /* layer */, + typename std::enable_if::value>::type* = 0) +{ + /* Nothing to do here */ +} + // Approximate Jacobian and supposedly-true Jacobian, then compare them // similarly to before. template double JacobianTest(ModuleType& module, - arma::mat& input, - const double minValue = -2, - const double maxValue = -1, - const double perturbation = 1e-6) + arma::mat& input, + const double minValue = -2, + const double maxValue = -1, + const double perturbation = 1e-6) { arma::mat output, outputA, outputB, jacobianA, jacobianB; - output.set_size(module.OutputSize(), input.n_cols); - outputA.set_size(module.OutputSize(), input.n_cols); - outputB.set_size(module.OutputSize(), input.n_cols); // Initialize the input matrix. RandomInitialization init(minValue, maxValue); init.Initialize(input, input.n_rows, input.n_cols); + // Initialize the module parameters. + ResetFunction(module); + // Initialize the jacobian matrix. module.Forward(input, output); jacobianA = arma::zeros(input.n_elem, output.n_elem); @@ -72,7 +89,7 @@ double JacobianTest(ModuleType& module, deriv.zeros(); derivTemp(i) = 1; - arma::mat delta(input.n_rows, input.n_cols); + arma::mat delta; module.Backward(input, deriv, delta); jacobianB.col(i) = delta; @@ -90,9 +107,9 @@ double CustomJacobianTest(ModuleType& module, const double perturbation = 1e-6) { arma::mat output, outputA, outputB, jacobianA, jacobianB; - output.set_size(module.OutputSize(), input.n_cols); - outputA.set_size(module.OutputSize(), input.n_cols); - outputB.set_size(module.OutputSize(), input.n_cols); + + // Initialize the module parameters. + ResetFunction(module); // Initialize the jacobian matrix. module.Forward(input, output); @@ -123,7 +140,7 @@ double CustomJacobianTest(ModuleType& module, deriv.zeros(); deriv(i) = 1; - arma::mat delta(input.n_rows, input.n_cols); + arma::mat delta; module.Backward(input, deriv, delta); jacobianB.col(i) = delta; diff --git a/src/mlpack/tests/ann_visitor_test.cpp b/src/mlpack/tests/ann_visitor_test.cpp new file mode 100644 index 0000000000..29f376611e --- /dev/null +++ b/src/mlpack/tests/ann_visitor_test.cpp @@ -0,0 +1,235 @@ +/** + * @file tests/ann_visitor_test.cpp + * + * Tests for testing visitors in ANN's of mlpack. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include +#include +#include + +#include "catch.hpp" +#include "test_catch_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; + +/** + * Test that the BiasSetVisitor works properly. + */ +TEST_CASE("BiasSetVisitorTest", "[ANNVisitorTest]") +{ + LayerTypes<> linear = new Linear<>(10, 10); + + arma::mat layerWeights(110, 1); + layerWeights.zeros(); + + ResetVisitor resetVisitor; + + boost::apply_visitor(WeightSetVisitor(layerWeights, 0), linear); + + boost::apply_visitor(resetVisitor, linear); + + arma::mat weight = {"1 2 3 4 5 6 7 8 9 10"}; + + size_t biasSize = boost::apply_visitor(BiasSetVisitor(weight, 0), linear); + + REQUIRE(biasSize == 10); + + arma::mat input(10, 1), output; + input.randu(); + + boost::apply_visitor(ForwardVisitor(input, output), linear); + + REQUIRE(arma::accu(output) == 55); + + boost::apply_visitor(DeleteVisitor(), linear); +} + +/** + * Check correctness of WeightSize() for a layer. + */ +void CheckCorrectnessOfWeightSize(LayerTypes<>& layer) +{ + size_t weightSize = boost::apply_visitor(WeightSizeVisitor(), + layer); + + arma::mat parameters; + boost::apply_visitor(ParametersVisitor(parameters), layer); + + REQUIRE(weightSize == parameters.n_elem); +} + +/** + * Test that WeightSetVisitor works properly. + */ +TEST_CASE("WeightSetVisitorTest", "[ANNVisitorTest]") +{ + size_t randomSize = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> linear = new Linear<>(randomSize, randomSize); + + arma::mat layerWeights(randomSize * randomSize + randomSize, 1); + layerWeights.zeros(); + + size_t setWeights = boost::apply_visitor(WeightSetVisitor(layerWeights, 0), + linear); + + REQUIRE(setWeights == randomSize * randomSize + randomSize); +} + +/** + * Test that WeightSizeVisitor works properly for linear layer. + */ +TEST_CASE("WeightSizeVisitorTestForLinearLayer", "[ANNVisitorTest]") +{ + size_t randomInSize = arma::randi(arma::distr_param(1, 100)); + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> linearLayer = new Linear<>(randomInSize, randomOutSize); + + CheckCorrectnessOfWeightSize(linearLayer); +} + +/** + * Test that WeightSizeVisitor works properly for concat layer. + */ +TEST_CASE("WeightSizeVisitorTestForConcatLayer", "[ANNVisitorTest]") +{ + LayerTypes<> concatLayer = new Concat<>(); + + CheckCorrectnessOfWeightSize(concatLayer); +} + +/** + * Test that WeightSizeVisitor works properly for fast lstm layer. + */ +TEST_CASE("WeightSizeVisitorTestForFastLSTMLayer", "[ANNVisitorTest]") +{ + size_t randomInSize = arma::randi(arma::distr_param(1, 100)); + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> fastLSTMLayer = new FastLSTM<>(randomInSize, randomOutSize); + + CheckCorrectnessOfWeightSize(fastLSTMLayer); +} + +/** + * Test that WeightSizeVisitor works properly for Add layer. + */ +TEST_CASE("WeightSizeVisitorTestForAddLayer", "[ANNVisitorTest]") +{ + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> addLayer = new Add<>(randomOutSize); + + CheckCorrectnessOfWeightSize(addLayer); +} + +/** + * Test that WeightSizeVisitor works properly for Atrous Convolution Layer. + */ +TEST_CASE("WeightSizeVisitorTestForAtrousConvolutionLayer", "[ANNVisitorTest]") +{ + size_t randomInSize = arma::randi(arma::distr_param(1, 100)); + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + size_t randomKernelWidth = arma::randi(arma::distr_param(1, 100)); + size_t randomKernelHeight = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> atrousConvLayer = new AtrousConvolution<>(randomInSize, + randomOutSize, randomKernelWidth, randomKernelHeight); + + CheckCorrectnessOfWeightSize(atrousConvLayer); +} + + +/** + * Test that WeightSizeVisitor works properly for Convolution layer. + */ +TEST_CASE("WeightSizeVisitorTestForConvLayer", "[ANNVisitorTest]") +{ + size_t randomInSize = arma::randi(arma::distr_param(1, 100)); + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + size_t randomKernelWidth = arma::randi(arma::distr_param(1, 100)); + size_t randomKernelHeight = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> convLayer = new Convolution<>(randomInSize, randomOutSize, + randomKernelWidth, randomKernelHeight); + CheckCorrectnessOfWeightSize(convLayer); +} + +/** + * Test that WeightSizeVisitor works properly for BatchNorm layer. + */ +TEST_CASE("WeightSizeVisitorTestForBatchNormLayer", "[ANNVisitorTest]") +{ + size_t randomSize = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> batchNorm = new BatchNorm<>(randomSize); + CheckCorrectnessOfWeightSize(batchNorm); +} + +/** + * Test that WeightSizeVisitor works properly for LSTM layer. + */ +TEST_CASE("WeightSizeVisitorTestForLSTMLayer", "[ANNVisitorTest]") +{ + size_t randomInSize = arma::randi(arma::distr_param(1, 100)); + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> lstm = new LSTM<>(randomInSize, randomOutSize); + CheckCorrectnessOfWeightSize(lstm); +} + +/** + * Test that WeightSizeVisitor works properly for Transposed Convolution layer. + */ +TEST_CASE("WeightSizeVisitorTestForTransposedConvLayer", "[ANNVisitorTest]") +{ + size_t randomInSize = arma::randi(arma::distr_param(1, 100)); + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + size_t randomKernelWidth = arma::randi(arma::distr_param(1, 100)); + size_t randomKernelHeight = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> transposedConvLayer = new TransposedConvolution<>(randomInSize, + randomOutSize, randomKernelWidth, randomKernelHeight); + + CheckCorrectnessOfWeightSize(transposedConvLayer); +} + +/** + * Test that WeightSizeVisitor works properly for noisy linear layer. + */ +TEST_CASE("WeightSizeVisitorTestForNoisyLinearLayer", "[ANNVisitorTest]") +{ + size_t randomInSize = arma::randi(arma::distr_param(1, 100)); + size_t randomOutSize = arma::randi(arma::distr_param(1, 100)); + + LayerTypes<> noisyLinearLayer = new NoisyLinear<>(randomInSize, + randomOutSize); + + CheckCorrectnessOfWeightSize(noisyLinearLayer); +} + +/** + * Test that WeightSizeVisitor works properly for Multihead Attention layer. + */ +TEST_CASE("WeightSizeVisitorTestForMultiheadAttentionLayer", "[ANNVisitorTest]") +{ + size_t randomtgtSeqLen = arma::randi(arma::distr_param(1, 100)); + size_t randomsrcSeqLen = arma::randi(arma::distr_param(1, 100)); + size_t randomembedDim = 768; + size_t randomnumHeads = 12; + + LayerTypes<> MultiheadAttentionLayer = new MultiheadAttention<>( + randomtgtSeqLen, randomsrcSeqLen, randomembedDim, randomnumHeads); + + CheckCorrectnessOfWeightSize(MultiheadAttentionLayer); +} diff --git a/src/mlpack/tests/async_learning_test.cpp b/src/mlpack/tests/async_learning_test.cpp index d788386c4f..4e8f03e9e3 100644 --- a/src/mlpack/tests/async_learning_test.cpp +++ b/src/mlpack/tests/async_learning_test.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include #include @@ -48,11 +48,11 @@ TEST_CASE("OneStepQLearningTest", "[AsyncLearningTest]") // Set up the network. FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); - model.Add(20); - model.Add(); - model.Add(20); - model.Add(); - model.Add(2); + model.Add>(4, 20); + model.Add>(); + model.Add>(20, 20); + model.Add>(); + model.Add>(20, 2); // Set up the policy. using Policy = GreedyPolicy; @@ -124,11 +124,11 @@ TEST_CASE("OneStepSarsaTest", "[AsyncLearningTest]") // Set up the network. FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); - model.Add(20); - model.Add(); - model.Add(20); - model.Add(); - model.Add(2); + model.Add>(4, 20); + model.Add>(); + model.Add>(20, 20); + model.Add>(); + model.Add>(20, 2); // Set up the policy. using Policy = GreedyPolicy; @@ -199,11 +199,11 @@ TEST_CASE("NStepQLearningTest", "[AsyncLearningTest]") // Set up the network. FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); - model.Add(20); - model.Add(); - model.Add(20); - model.Add(); - model.Add(2); + model.Add>(4, 20); + model.Add>(); + model.Add>(20, 20); + model.Add>(); + model.Add>(20, 2); // Set up the policy. using Policy = GreedyPolicy; diff --git a/src/mlpack/tests/callback_test.cpp b/src/mlpack/tests/callback_test.cpp index 203ce3d9a5..aeeed3fe26 100644 --- a/src/mlpack/tests/callback_test.cpp +++ b/src/mlpack/tests/callback_test.cpp @@ -13,8 +13,7 @@ #include #include #include -#include -//#include +#include #include #include #include @@ -48,12 +47,12 @@ TEST_CASE("FFNCallbackTest", "[CallbackTest]") if (!data::Load("lab3.csv", labels)) FAIL("Cannot load test dataset lab3.csv!"); - FFN model; + FFN, RandomInitialization> model; - model.Add(2); - model.Add(); - model.Add(1); - model.Add(); + model.Add>(1, 2); + model.Add>(); + model.Add>(2, 1); + model.Add>(); std::stringstream stream; model.Train(data, labels, ens::PrintLoss(stream)); @@ -74,12 +73,12 @@ TEST_CASE("FFNWithOptimizerCallbackTest", "[CallbackTest]") if (!data::Load("lab3.csv", labels)) FAIL("Cannot load test dataset lab3.csv!"); - FFN model; + FFN, RandomInitialization> model; - model.Add(2); - model.Add(); - model.Add(1); - model.Add(); + model.Add>(1, 2); + model.Add>(); + model.Add>(2, 1); + model.Add>(); std::stringstream stream; ens::StandardSGD opt(0.1, 1, 5); @@ -99,13 +98,14 @@ TEST_CASE("RNNCallbackTest", "[CallbackTest]") RandomInitialization init(0.5, 0.5); // Create model with user defined rho parameter. - RNN model( - rho, false, NegativeLogLikelihood(), init); - model.Add(10); + RNN, RandomInitialization> model( + rho, false, NegativeLogLikelihood<>(), init); + model.Add>(); + model.Add>(1, 10); - // Use LSTM layer with 3 units. - model.Add(3); - model.Add(); + // Use LSTM layer with rho. + model.Add>(10, 3, rho); + model.Add>(); std::stringstream stream; model.Train(input, target, ens::PrintLoss(stream)); @@ -124,13 +124,14 @@ TEST_CASE("RNNWithOptimizerCallbackTest", "[CallbackTest]") RandomInitialization init(0.5, 0.5); // Create model with user defined rho parameter. - RNN model( - rho, false, NegativeLogLikelihood(), init); - model.Add(10); + RNN, RandomInitialization> model( + rho, false, NegativeLogLikelihood<>(), init); + model.Add>(); + model.Add>(1, 10); - // Use LSTM layer with 3 units. - model.Add(3); - model.Add(); + // Use LSTM layer with rho. + model.Add>(10, 3, rho); + model.Add>(); std::stringstream stream; ens::StandardSGD opt(0.1, 1, 5); @@ -233,7 +234,7 @@ TEST_CASE("SRWithOptimizerCallback", "[CallbackTest]") /* * Tests the RBM Implementation with PrintLoss callback. - * + */ TEST_CASE("RBMCallbackTest", "[CallbackTest]") { // Normalised dataset. @@ -258,7 +259,7 @@ TEST_CASE("RBMCallbackTest", "[CallbackTest]") double objVal = model.Train(msgd, ens::ProgressBar(70, stream)); REQUIRE(!std::isnan(objVal)); REQUIRE(stream.str().length() > 0); -}*/ +} /** * Tests the SparseAutoencoder implementation with diff --git a/src/mlpack/tests/catch.hpp b/src/mlpack/tests/catch.hpp index d2a12427b2..db1fed3b98 100644 --- a/src/mlpack/tests/catch.hpp +++ b/src/mlpack/tests/catch.hpp @@ -1,6 +1,6 @@ /* - * Catch v2.13.9 - * Generated: 2022-04-12 22:37:23.260201 + * Catch v2.13.8 + * Generated: 2022-01-03 21:20:09.589503 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly * Copyright (c) 2022 Two Blue Cubes Ltd. All rights reserved. @@ -15,7 +15,7 @@ #define CATCH_VERSION_MAJOR 2 #define CATCH_VERSION_MINOR 13 -#define CATCH_VERSION_PATCH 9 +#define CATCH_VERSION_PATCH 8 #ifdef __clang__ # pragma clang system_header @@ -13392,10 +13392,6 @@ namespace Catch { filename.erase(0, lastSlash); filename[0] = '#'; } - else - { - filename.insert(0, "#"); - } auto lastDot = filename.find_last_of('.'); if (lastDot != std::string::npos) { @@ -15391,7 +15387,7 @@ namespace Catch { } Version const& libraryVersion() { - static Version version( 2, 13, 9, "", 0 ); + static Version version( 2, 13, 8, "", 0 ); return version; } @@ -17894,7 +17890,7 @@ using Catch::Detail::Approx; #define INFO( msg ) (void)(0) #define UNSCOPED_INFO( msg ) (void)(0) #define WARN( msg ) (void)(0) -#define CAPTURE( ... ) (void)(0) +#define CAPTURE( msg ) (void)(0) #define TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ )) #define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ )) diff --git a/src/mlpack/tests/convolution_test.cpp b/src/mlpack/tests/convolution_test.cpp index 1893804aa5..c2798090ce 100644 --- a/src/mlpack/tests/convolution_test.cpp +++ b/src/mlpack/tests/convolution_test.cpp @@ -54,46 +54,6 @@ void Convolution2DMethodTest(const arma::mat input, REQUIRE(*outputPtr == Approx(*convOutputPtr).epsilon(1e-5)); } -/* - * Implementation of the convolution function test with custom stride and - * dilation. This does not work for every convolution type. - * - * @param input Input used to perform the convolution. - * @param filter Filter used to perform the convolution. - * @param output The reference output data that contains the results of the - * convolution. - * @param strideH Height stride parameter. - * @param strideW Width stride parameter. - * @param dilationH Height dilation parameter. - * @param dilationW Width dilation parameter. - * - * @tparam ConvolutionFunction Convolution function used for the check. - */ -template -void Convolution2DMethodTest(const arma::mat input, - const arma::mat filter, - const arma::mat output, - const size_t strideW, - const size_t strideH, - const size_t dilationW, - const size_t dilationH) -{ - arma::mat convOutput; - ConvolutionFunction::Convolution(input, filter, convOutput, strideW, strideH, - dilationW, dilationH); - - // Check the output dimension. - bool b = (convOutput.n_rows == output.n_rows) && - (convOutput.n_cols == output.n_cols); - REQUIRE(b == 1); - - const double* outputPtr = output.memptr(); - const double* convOutputPtr = convOutput.memptr(); - - for (size_t i = 0; i < output.n_elem; ++i, outputPtr++, convOutputPtr++) - REQUIRE(*outputPtr == Approx(*convOutputPtr).epsilon(1e-5)); -} - /* * Implementation of the convolution function test using 3rd order tensors. * @@ -408,163 +368,3 @@ TEST_CASE("FullConvolutionBatchTest", "[ConvolutionTest]") ConvolutionMethodBatchTest >(input, filterCube, outputCube); } - -/** - * Test that non-stride-1 convolution works the same as stride-1 convolution on - * a smaller matrix. - */ -TEST_CASE("Stride2ConvolutionTest", "[ConvolutionTest]") -{ - // Generate dataset. - arma::mat input, filter, output; - input = { { 1, 2, 3, 4 }, - { 4, 1, 2, 3 }, - { 3, 4, 1, 2 }, - { 2, 3, 4, 1 } }; - - filter = { { 1, -1 }, - { -1, 1 } }; - - output = { { 1, 1, -4 }, - { -1, -4, 1 }, - { -2, -1, 1 } }; - - // Perform the naive convolution approach. - Convolution2DMethodTest >(input, filter, - output, 2, 2, 1, 1); -} - -TEST_CASE("Stride3ConvolutionTest", "[ConvolutionTest]") -{ - // Generate dataset. - arma::mat input, filter, output; - input = { { 1, 2, 3, 4 }, - { 4, 1, 2, 3 }, - { 3, 4, 1, 2 }, - { 2, 3, 4, 1 } }; - - filter = { { 1, -1 }, - { -1, 1 } }; - - output = { { 1, 1 }, - { -1, -4 } }; - - // Perform the naive convolution approach. - Convolution2DMethodTest >(input, filter, - output, 3, 3, 1, 1); -} - -TEST_CASE("UnequalStrideConvolutionTest", "[ConvolutionTest]") -{ - // Generate dataset. - arma::mat input, filter, output; - input = { { 1, 2, 3, 4 }, - { 4, 1, 2, 3 }, - { 3, 4, 1, 2 }, - { 2, 3, 4, 1 } }; - - filter = { { 1, -1 }, - { -1, 1 } }; - - output = { { 1, 1 }, - { -1, 0 }, - { -2, 3 } }; - - // Perform the naive convolution approach. - Convolution2DMethodTest >(input, filter, - output, 3, 2, 1, 1); -} - -TEST_CASE("Dilation2ConvolutionTest", "[ConvolutionTest]") -{ - // Generate dataset. - arma::mat input, filter, output; - input = { { 1, 2, 3, 4 }, - { 4, 1, 2, 3 }, - { 3, 4, 1, 2 }, - { 2, 3, 4, 1 } }; - - filter = { { 1, -1 }, - { -1, 1 } }; - - output = { { 1, 2, 2, 2, -3, -4 }, - { 4, 1, -2, 2, -2, -3 }, - { 2, 2, -4, -4, 2, 2 }, - { -2, 2, 4, -4, -2, 2 }, - { -3, -4, 2, 2, 1, 2 }, - { -2, -3, -2, 2, 4, 1 } }; - - // Perform the naive convolution approach. - Convolution2DMethodTest >(input, filter, - output, 1, 1, 2, 2); -} - -TEST_CASE("Dilation3ConvolutionTest", "[ConvolutionTest]") -{ - // Generate dataset. - arma::mat input, filter, output; - input = { { 1, 2, 3, 4 }, - { 4, 1, 2, 3 }, - { 3, 4, 1, 2 }, - { 2, 3, 4, 1 } }; - - filter = { { 1, -1 }, - { -1, 1 } }; - - output = { { 1, 2, 3, 3, -2, -3, -4 }, - { 4, 1, 2, -1, -1, -2, -3 }, - { 3, 4, 1, -1, -4, -1, -2 }, - { 1, 1, 1, -4, -1, -1, 3 }, - { -4, -1, -2, 1, 1, 2, 3 }, - { -3, -4, -1, 1, 4, 1, 2 }, - { -2, -3, -4, 1, 3, 4, 1 } }; - - // Perform the naive convolution approach. - Convolution2DMethodTest >(input, filter, - output, 1, 1, 3, 3); -} - -TEST_CASE("UnequalDilationConvolutionTest", "[ConvolutionTest]") -{ - // Generate dataset. - arma::mat input, filter, output; - input = { { 1, 2, 3, 4 }, - { 4, 1, 2, 3 }, - { 3, 4, 1, 2 }, - { 2, 3, 4, 1 } }; - - filter = { { 1, -1 }, - { -1, 1 } }; - - output = { { 1, 2, 3, 3, -2, -3, -4 }, - { 4, 1, 2, -1, -1, -2, -3 }, - { 2, 2, -2, -4, -2, 2, 2 }, - { -2, 2, 2, 0, -2, -2, 2 }, - { -3, -4, -1, 1, 4, 1, 2 }, - { -2, -3, -4, 1, 3, 4, 1 } }; - - // Perform the naive convolution approach. - Convolution2DMethodTest >(input, filter, - output, 1, 1, 3, 2); -} - -TEST_CASE("DilationAndStrideConvolutionTest", "[ConvolutionTest]") -{ - // Generate dataset. - arma::mat input, filter, output; - input = { { 1, 2, 3, 4 }, - { 4, 1, 2, 3 }, - { 3, 4, 1, 2 }, - { 2, 3, 4, 1 } }; - - filter = { { 1, -1 }, - { -1, 1 } }; - - output = { { 1, 2, -3 }, - { 2, -4, 2 }, - { -3, 2, 1 } }; - - // Perform the naive convolution approach. - Convolution2DMethodTest >(input, filter, - output, 2, 2, 2, 2); -} diff --git a/src/mlpack/tests/convolutional_network_test.cpp b/src/mlpack/tests/convolutional_network_test.cpp index 91994e6b8c..43eb1bb0d2 100644 --- a/src/mlpack/tests/convolutional_network_test.cpp +++ b/src/mlpack/tests/convolutional_network_test.cpp @@ -14,7 +14,6 @@ #include #include -#include #include #include @@ -71,84 +70,6 @@ void CheckMoveFunction(ModelType* network1, CheckMatrices(predictions1, predictions2); } -/** - * Build a trivial network with a single padding layer, and make sure it - * successfully pads the input. - */ -TEST_CASE("PaddingTest", "[ConvolutionalNetworktest]") -{ - arma::mat X; - X.load("mnist_first250_training_4s_and_9s.arm"); - - // Create the network. - FFN model; - - model.Add(1, 2, 3, 4); - - // Now, pass the data through. - arma::mat results; - model.InputDimensions() = std::vector({ 28, 28 }); - model.Forward(X, results); - - // Ensure that things are correctly padded. - arma::cube reshapedResults(results.memptr(), 35, 31, results.n_cols, false, - true); - - for (size_t i = 0; i < reshapedResults.n_slices; ++i) - { - // Check left. - for (size_t j = 0; j < reshapedResults.n_rows; ++j) - REQUIRE(reshapedResults(j, 0, i) == 0.0); - - // Check top. - for (size_t j = 0; j < 3; ++j) - for (size_t k = 0; k < reshapedResults.n_cols; ++k) - REQUIRE(reshapedResults(j, k, i) == 0.0); - - // Check bottom. - for (size_t j = 31; j < reshapedResults.n_rows; ++j) - for (size_t k = 0; k < reshapedResults.n_cols; ++k) - REQUIRE(reshapedResults(j, k, i) == 0.0); - - // Check right. - for (size_t j = 0; j < reshapedResults.n_rows; ++j) - for (size_t k = 29; k < reshapedResults.n_cols; ++k) - REQUIRE(reshapedResults(j, k, i) == 0.0); - } -} - -/** - * Build a trivial network with a MaxPooling layer, and make sure it - * successfully does the max-pool operation. - */ -TEST_CASE("MaxPoolingTest", "[ConvolutionalNetworkTest]") -{ - arma::mat X(8, 3); - X.col(0) = arma::vec("1, 2, 3, 4, 5, 6, 7, 8"); - X.col(1) = arma::vec("5, 7, 6, 8, 4, 3, 1, 2"); - X.col(2) = arma::vec("3, 4, 1, -1, 5, 5, 5, 5"); - - // Create the network. - FFN model; - model.Add(2, 2); - - arma::mat results; - model.InputDimensions() = std::vector({ 2, 4 }); - model.Forward(X, results); - - REQUIRE(results.n_rows == 3); - REQUIRE(results.n_cols == 3); - REQUIRE(results(0, 0) == 4); - REQUIRE(results(1, 0) == 6); - REQUIRE(results(2, 0) == 8); - REQUIRE(results(0, 1) == 8); - REQUIRE(results(1, 1) == 8); - REQUIRE(results(2, 1) == 4); - REQUIRE(results(0, 2) == 4); - REQUIRE(results(1, 2) == 5); - REQUIRE(results(2, 2) == 5); -} - /** * Train the vanilla network on a larger dataset. */ @@ -204,27 +125,25 @@ TEST_CASE("VanillaNetworkTest", "[ConvolutionalNetworkTest]") bool success = false; for (size_t trial = 0; trial < 5; ++trial) { - FFN model; + FFN, RandomInitialization> model; - model.Add(8, 5, 5, 1, 1, 0, 0); - model.Add(); - model.Add(2, 2); - model.Add(12, 2, 2); - model.Add(); - model.Add(2, 2); - model.Add(20); - model.Add(); - model.Add(10); - model.Add(); - model.Add(2); - model.Add(); - - model.InputDimensions() = std::vector({ 28, 28 }); + model.Add >(1, 8, 5, 5, 1, 1, 0, 0, 28, 28); + model.Add >(); + model.Add >(8, 8, 2, 2); + model.Add >(8, 12, 2, 2); + model.Add >(); + model.Add >(2, 2, 2, 2); + model.Add >(192, 20); + model.Add >(); + model.Add >(20, 10); + model.Add >(); + model.Add >(10, 2); + model.Add >(); // Train for only 8 epochs. ens::RMSProp opt(0.001, 1, 0.88, 1e-8, 8 * nPoints, -1); - double objVal = model.Train(X, Y, opt, ens::PrintLoss()); + double objVal = model.Train(X, Y, opt); // Test that objective value returned by FFN::Train() is finite. REQUIRE(std::isfinite(objVal) == true); @@ -251,103 +170,6 @@ TEST_CASE("VanillaNetworkTest", "[ConvolutionalNetworkTest]") REQUIRE(success == true); } -TEST_CASE("VanillaNetworkBatchSizeTest", "[ConvolutionalNetworkTest]") -{ - FFN model; - - model.Add(8, 5, 5, 1, 1, 0, 0); - model.Add(); - model.Add(2, 2); - model.Add(12, 2, 2); - model.Add(); - model.Add(2, 2); - model.Add(20); - model.Add(); - model.Add(10); - model.Add(); - model.Add(2); - model.Add(); - - model.InputDimensions() = std::vector({ 28, 28 }); - - arma::mat X; - X.load("mnist_first250_training_4s_and_9s.arm"); - - // Normalize each point since these are images. - arma::uword nPoints = X.n_cols; - for (arma::uword i = 0; i < nPoints; ++i) - { - X.col(i) /= norm(X.col(i), 2); - } - - // Build the target matrix. - arma::mat Y = arma::zeros(1, nPoints); - for (size_t i = 0; i < nPoints; ++i) - { - if (i < nPoints / 2) - { - // Assign label "1" to all samples with digit = 4 - Y(i) = 1; - } - else - { - // Assign label "0" to all samples with digit = 9 - Y(i) = 0; - } - } - - // Perform one epoch of training to get the weights to somewhere reasonable. - ens::RMSProp opt(0.001, 1, 0.88, 1e-8, nPoints, -1); - model.Train(X, Y, opt); - - size_t trials = 7; - for (size_t trial = 0; trial < trials; ++trial) - { - const size_t batchSize = std::pow(2.0, (double) trial + 1.0); - - // Check the forward pass, and then call EvaluateWithGradient() to compute - // the gradient. - arma::mat results; - arma::mat batchData = X.cols(0, batchSize - 1); - arma::mat batchResponses = Y.cols(0, batchSize - 1); - model.ResetData(std::move(batchData), std::move(batchResponses)); - model.Forward(X.cols(0, batchSize - 1), results); - - arma::mat gradient(1, model.WeightSize()); - const double obj = model.EvaluateWithGradient(model.Parameters(), gradient); - - REQUIRE(results.n_cols == batchSize); - - // Now compute results with a batch size of 1. - arma::mat singleResults(results.n_rows, results.n_cols); - arma::mat singleGradient(gradient.n_rows, gradient.n_cols); - double singleObj = 0.0; - - for (size_t i = 0; i < batchSize; ++i) - { - arma::mat tmpResult; - arma::mat singleData = X.cols(i, i); - arma::mat singleResponses = Y.cols(i, i); - model.ResetData(std::move(singleData), std::move(singleResponses)); - model.Forward(X.cols(i, i), tmpResult); - REQUIRE(tmpResult.n_cols == 1); - singleResults.col(i) = tmpResult; - - arma::mat tmpGradient(1, model.WeightSize()); - singleObj += model.EvaluateWithGradient(model.Parameters(), tmpGradient); - - singleGradient += tmpGradient; - } - - // Check the forward pass results. - CheckMatrices(results, singleResults); - - // Now, check EvaluateWithGradient()'s results. - REQUIRE(obj == Approx(singleObj)); - CheckMatrices(gradient, singleGradient); - } -} - /** * Train the vanilla network on a larger dataset. */ @@ -400,132 +222,39 @@ TEST_CASE("CheckCopyVanillaNetworkTest", "[ConvolutionalNetworkTest]") // of iterations using random weights. If this works 1 of 5 times, I'm fine // with that. All I want to know is that the network is able to escape from // local minima and to solve the task. - FFN *model = - new FFN; + FFN, RandomInitialization> *model = new FFN, RandomInitialization>; - model->Add(8, 5, 5, 1, 1, 0, 0); - model->Add(); - model->Add(2, 2); - model->Add(12, 2, 2); - model->Add(); - model->Add(2, 2); - model->Add(20); - model->Add(); - model->Add(10); - model->Add(); - model->Add(2); - model->Add(); - model->InputDimensions() = std::vector({ 28, 28 }); + model->Add >(1, 8, 5, 5, 1, 1, 0, 0, 28, 28); + model->Add >(); + model->Add >(8, 8, 2, 2); + model->Add >(8, 12, 2, 2); + model->Add >(); + model->Add >(2, 2, 2, 2); + model->Add >(192, 20); + model->Add >(); + model->Add >(20, 10); + model->Add >(); + model->Add >(10, 2); + model->Add >(); - FFN *model1 = - new FFN; - - model1->Add(8, 5, 5, 1, 1, 0, 0); - model1->Add(); - model1->Add(2, 2); - model1->Add(12, 2, 2); - model1->Add(); - model1->Add(2, 2); - model1->Add(20); - model1->Add(); - model1->Add(10); - model1->Add(); - model1->Add(2); - model1->Add(); - model1->InputDimensions() = std::vector({ 28, 28 }); + FFN, RandomInitialization> *model1 = new FFN, RandomInitialization>; + model1->Add >(1, 8, 5, 5, 1, 1, 0, 0, 28, 28); + model1->Add >(); + model1->Add >(8, 8, 2, 2); + model1->Add >(8, 12, 2, 2); + model1->Add >(); + model1->Add >(2, 2, 2, 2); + model1->Add >(192, 20); + model1->Add >(); + model1->Add >(20, 10); + model1->Add >(); + model1->Add >(10, 2); + model1->Add >(); + // Check whether copy constructor is working or not. CheckCopyFunction<>(model, X, Y, 8); // Check whether move constructor is working or not. CheckMoveFunction<>(model1, X, Y, 8); } - -TEST_CASE("Issue2986", "[ConvolutionalNetworkTest]") -{ - // Ensure that the code snippet in issue #2986 succeeds without any issues. - arma::mat input, output, delta; - input.ones(36, 1); - - // Note that the stride here is 2, not 1. - Convolution c(1, 3, 3, 2, 2, 0, 0); - - // Set up the layer without an enclosing FFN. - c.InputDimensions() = std::vector({ 6, 6 }); - c.ComputeOutputDimensions(); - arma::mat weights(c.WeightSize(), 1, arma::fill::randu); - c.SetWeights(weights.memptr()); - - output.set_size(c.OutputSize(), 1); - delta.set_size(input.size()); - - REQUIRE_NOTHROW(c.Forward(input, output)); - REQUIRE_NOTHROW(c.Backward(input, output, delta)); - - // Now test with a stride of 3. - c = Convolution(1, 3, 3, 3, 3, 0, 0); - - // Set up the layer without an enclosing FFN. - c.InputDimensions() = std::vector({ 6, 6 }); - c.ComputeOutputDimensions(); - weights.set_size(c.WeightSize(), 1); - weights.randu(); - c.SetWeights(weights.memptr()); - - output.set_size(c.OutputSize(), 1); - delta.set_size(input.size()); - - REQUIRE_NOTHROW(c.Forward(input, output)); - REQUIRE_NOTHROW(c.Backward(input, output, delta)); - - // Now test with different strides for height and width. - c = Convolution(1, 3, 3, 2, 3, 0, 0); - - // Set up the layer without an enclosing FFN. - c.InputDimensions() = std::vector({ 6, 6 }); - c.ComputeOutputDimensions(); - weights.set_size(c.WeightSize(), 1); - weights.randu(); - c.SetWeights(weights.memptr()); - - output.set_size(c.OutputSize(), 1); - delta.set_size(input.size()); - - REQUIRE_NOTHROW(c.Forward(input, output)); - REQUIRE_NOTHROW(c.Backward(input, output, delta)); -} - -// Test that the Convolution layer gives reasonable output when a non-zero -// padding size is used. -TEST_CASE("CustomPaddingTest", "[ConvolutionalNetworkTest]") -{ - arma::mat input, output, delta, weights; - input.ones(36, 1); - - Convolution c = Convolution(1, 3, 3, 1, 1, { 1, 2 }, { 3, 4 }, "none"); - - c.InputDimensions() = std::vector({ 6, 6 }); - c.ComputeOutputDimensions(); - - // First, check that the output dimensions are reasonable. - REQUIRE(c.OutputDimensions().size() == 3); - REQUIRE(c.OutputDimensions()[0] == 7); - REQUIRE(c.OutputDimensions()[1] == 11); - REQUIRE(c.OutputDimensions()[2] == 1); - - weights.set_size(c.WeightSize(), 1); - weights.ones(); - c.SetWeights(weights.memptr()); - - // Now make sure that the forward pass returns the correct output. - output.set_size(c.OutputSize(), 1); - REQUIRE_NOTHROW(c.Forward(input, output)); - REQUIRE(output.n_rows == c.OutputSize()); - REQUIRE(output.n_cols == 1); - // The lower right corner's convolution entry should only touch one input - // value (and everything else padding). - REQUIRE(output(output.n_rows - 1, 0) == 1.0); - - delta.set_size(input.size()); - REQUIRE_NOTHROW(c.Backward(input, output, delta)); -} diff --git a/src/mlpack/tests/custom_layer.hpp b/src/mlpack/tests/custom_layer.hpp index 3a3eaa78ad..593496bb50 100644 --- a/src/mlpack/tests/custom_layer.hpp +++ b/src/mlpack/tests/custom_layer.hpp @@ -2,7 +2,7 @@ * @file tests/custom_layer.hpp * @author Projyal Dev * - * A simple custom layer mimicking SigmoidLayer for testing if custom + * A simple custom layer mimicing SigmoidLayer for testing if custom * layers work. * * mlpack is free software; you may redistribute it and/or modify it under the @@ -15,20 +15,20 @@ #include #include -#include -#include + namespace mlpack { namespace ann { - -/** - * Standard Sigmoid layer. - */ -template < - class ActivationFunction = LogisticFunction, - typename MatType = arma::mat -> -using CustomLayer = BaseLayer; + /** + * Standard Sigmoid layer. + */ + template < + class ActivationFunction = LogisticFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat + > + using CustomLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; } // namespace ann } // namespace mlpack diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 1ee34b0566..aafee5db11 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -45,7 +44,6 @@ using namespace mlpack; using namespace mlpack::ann; using namespace mlpack::cv; using namespace mlpack::naive_bayes; -using namespace mlpack::perceptron; using namespace mlpack::regression; using namespace mlpack::tree; using namespace mlpack::data; @@ -303,9 +301,10 @@ TEST_CASE("MSEMatResponsesTest", "[CVTest]") arma::mat data("1 2"); arma::mat trainingResponses("1 2; 3 4"); - FFN ffn(MeanSquaredError(), + FFN, ConstInitialization> ffn(MeanSquaredError<>(), ConstInitialization(0)); - ffn.Add(2); + ffn.Add>(1, 2); + ffn.Add>(); ens::RMSProp opt(0.2); opt.BatchSize() = 1; @@ -613,33 +612,6 @@ TEST_CASE("KFoldCVAccuracyTest", "[CVTest]") REQUIRE_NOTHROW(cv.Model()); } -/** - * Test k-fold cross-validation with the perceptron. - */ -TEST_CASE("KFoldCVPerceptronTest", "[CVTest]") -{ - // The same as the test above (for Naive Bayes), but with the perceptron. - - // Making a 10-points dataset. The last point should be classified wrong when - // it is tested separately. - arma::mat data("0 1 2 3 100 101 102 103 104 5"); - arma::Row labels("0 0 0 0 1 1 1 1 1 1"); - size_t numClasses = 2; - - // 10-fold cross-validation, no shuffling. - KFoldCV, Accuracy> cv(10, data, labels, numClasses); - - // We should succeed in classifying separately the first nine samples, and - // fail with the remaining one. - double expectedAccuracy = (9 * 1.0 + 0.0) / 10; - - REQUIRE(cv.Evaluate() == Approx(expectedAccuracy).epsilon(1e-7)); - - // Assert we can access a trained model without the exception of - // uninitialization. - REQUIRE_NOTHROW(cv.Model()); -} - /** * Test k-fold cross-validation with weighted linear regression. */ diff --git a/src/mlpack/tests/feedforward_network_2_test.cpp b/src/mlpack/tests/feedforward_network_2_test.cpp index a115468305..90ba3c3fbd 100644 --- a/src/mlpack/tests/feedforward_network_2_test.cpp +++ b/src/mlpack/tests/feedforward_network_2_test.cpp @@ -14,7 +14,6 @@ #include #include -#include #include #include #include @@ -102,9 +101,9 @@ TEST_CASE("RBFNetworkTest", "[FeedForwardNetworkTest]") KMeans<> kmeans; kmeans.Cluster(trainData, 8, centroids); - FFN model; - model.Add(8, centroids); - model.Add(3); + FFN > model; + model.Add >(trainData.n_rows, 8, centroids); + model.Add >(8, 3); // RBFN neural net with MeanSquaredError. TestNetwork<>(model, trainData, trainLabels1, testData, testLabels, 10, 0.1); @@ -134,9 +133,9 @@ TEST_CASE("RBFNetworkTest", "[FeedForwardNetworkTest]") KMeans<> kmeans1; kmeans1.Cluster(dataset, 140, centroids1); - FFN model1; - model1.Add(140, centroids1, 4.1); - model1.Add(2); + FFN > model1; + model1.Add >(dataset.n_rows, 140, centroids1, 4.1); + model1.Add >(140, 2); // RBFN neural net with MeanSquaredError. TestNetwork<>(model1, dataset, labels1, dataset, labels, 10, 0.1); diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index 613d0e56e8..6fd5af4e30 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -12,7 +12,7 @@ */ #include -#include +#include #include #include @@ -20,7 +20,7 @@ #include "catch.hpp" #include "serialization.hpp" -//#include "custom_layer.hpp" +#include "custom_layer.hpp" using namespace mlpack; using namespace mlpack::ann; @@ -37,7 +37,7 @@ void TestNetwork(ModelType& model, const size_t maxEpochs, const double classificationErrorThreshold) { - ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols * maxEpochs, -100); + ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1); model.Train(trainData, trainLabels, opt); MatType predictionTemp; @@ -51,7 +51,6 @@ void TestNetwork(ModelType& model, } size_t correct = arma::accu(prediction == testLabels); - double classificationError = 1 - double(correct) / testData.n_cols; REQUIRE(classificationError <= classificationErrorThreshold); } @@ -60,14 +59,14 @@ void TestNetwork(ModelType& model, template void CheckCopyFunction(ModelType* network1, MatType& trainData, - MatType& trainLabels) + MatType& trainLabels, + const size_t maxEpochs) { - ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols, -1); + ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1); network1->Train(trainData, trainLabels, opt); arma::mat predictions1; network1->Predict(trainData, predictions1); - FFN<> network2; network2 = *network1; delete network1; @@ -86,7 +85,7 @@ void CheckMoveFunction(ModelType* network1, MatType& trainLabels, const size_t maxEpochs) { - ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols, -1); + ens::RMSProp opt(0.01, 32, 0.88, 1e-8, maxEpochs * trainData.n_cols, -1); network1->Train(trainData, trainLabels, opt); arma::mat predictions1; @@ -137,28 +136,28 @@ TEST_CASE("CheckCopyMovingVanillaNetworkTest", "[FeedForwardNetworkTest]") * +-----+ +-----+ */ - FFN *model = new FFN; - model->Add(8); - model->Add(); - model->Add(3); - model->Add(); + FFN > *model = new FFN >; + model->Add >(trainData.n_rows, 8); + model->Add >(); + model->Add >(8, 3); + model->Add >(); - FFN *model1 = new FFN; - model1->Add(8); - model1->Add(); - model1->Add(3); - model1->Add(); + FFN > *model1 = new FFN >; + model1->Add >(trainData.n_rows, 8); + model1->Add >(); + model1->Add >(8, 3); + model1->Add >(); // Check whether copy constructor is working or not. - CheckCopyFunction(model, trainData, trainLabels); + CheckCopyFunction<>(model, trainData, trainLabels, 1); // Check whether move constructor is working or not. - CheckMoveFunction(model1, trainData, trainLabels, 1); + CheckMoveFunction<>(model1, trainData, trainLabels, 1); } /** * Check whether copying and moving network with Reparametrization is working or not. - * + */ TEST_CASE("CheckCopyMovingReparametrizationNetworkTest", "[FeedForwardNetworkTest]") { @@ -166,64 +165,31 @@ TEST_CASE("CheckCopyMovingReparametrizationNetworkTest", arma::mat trainData; data::Load("thyroid_train.csv", trainData, true); - arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + // Normalize labels to [0, 2]. + arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; trainData.shed_row(trainData.n_rows - 1); - // Construct a feed forward network with trainData.n_rows input nodes, - // followed by a linear layer and then a reparametrization layer. - FFN *model = new FFN; - model->Add(8); - model->Add(false, true, 1); - model->Add(); + /* + * Construct a feed forward network with trainData.n_rows input nodes, + * followed by a linear layer and then a reparametrization layer. + */ - FFN *model1 = new FFN; - model1->Add(8); - model1->Add(false, true, 1); - model1->Add(); + FFN > *model = new FFN >; + model->Add >(trainData.n_rows, 8); + model->Add >(4, false, true, 1); + model->Add >(); + + FFN > *model1 = new FFN >; + model1->Add >(trainData.n_rows, 8); + model1->Add >(4, false, true, 1); + model1->Add >(); // Check whether copy constructor is working or not. - CheckCopyFunction(model, trainData, trainLabels); + CheckCopyFunction<>(model, trainData, trainLabels, 1); // Check whether move constructor is working or not. - CheckMoveFunction(model1, trainData, trainLabels, 1); + CheckMoveFunction<>(model1, trainData, trainLabels, 1); } -*/ - -/** - * Check whether copying and moving network with Reparametrization is working or not. - */ -// TEST_CASE("CheckCopyMovingReparametrizationNetworkTest", -// "[FeedForwardNetworkTest]") -// { -// // Load the dataset. -// arma::mat trainData; -// data::Load("thyroid_train.csv", trainData, true); -// -// // Normalize labels to [0, 2]. -// arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; -// trainData.shed_row(trainData.n_rows - 1); -// -// /* -// * Construct a feed forward network with trainData.n_rows input nodes, -// * followed by a linear layer and then a reparametrization layer. -// */ -// -// FFN *model = new FFN; -// model->Add >(trainData.n_rows, 8); -// model->Add >(4, false, true, 1); -// model->Add >(); -// -// FFN *model1 = new FFN; -// model1->Add >(trainData.n_rows, 8); -// model1->Add >(4, false, true, 1); -// model1->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckCopyFunction<>(model, trainData, trainLabels, 1); -// -// // Check whether move constructor is working or not. -// CheckMoveFunction<>(model1, trainData, trainLabels, 1); -// } /** * Check whether copying and moving network with linear3d is working or not. @@ -238,25 +204,45 @@ TEST_CASE("CheckCopyMovingLinear3DNetworkTest", "[FeedForwardNetworkTest]") arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; trainData.shed_row(trainData.n_rows - 1); - // Construct a feed forward network with trainData.n_rows input nodes, - // followed by a linear layer and then a Linear3D layer. - FFN *model = new FFN; - model->Add(8); - model->Add(); - model->Add(3); - model->Add(); + /* + * Construct a feed forward network with trainData.n_rows input nodes, + * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The + * network structure looks like: + * + * Input Hidden Output + * Layer Layer Layer + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | +>| | +>| | + * +-----+ | +--+--+ | +-----+ + * | | + * Bias | Bias | + * Layer | Layer | + * +-----+ | +-----+ | + * | | | | | | + * | +-----+ | +-----+ + * | | | | + * +-----+ +-----+ + */ - FFN *model1 = new FFN; - model1->Add(8); - model1->Add(); - model1->Add(3); - model1->Add(); + FFN > *model = new FFN >; + model->Add >(trainData.n_rows, 8); + model->Add >(); + model->Add >(8, 3); + model->Add >(); + + FFN > *model1 = new FFN >; + model1->Add >(trainData.n_rows, 8); + model1->Add >(); + model1->Add >(8, 3); + model1->Add >(); // Check whether copy constructor is working or not. - CheckCopyFunction(model, trainData, trainLabels); + CheckCopyFunction<>(model, trainData, trainLabels, 1); // Check whether move constructor is working or not. - CheckMoveFunction(model1, trainData, trainLabels, 1); + CheckMoveFunction<>(model1, trainData, trainLabels, 1); } /** @@ -270,24 +256,28 @@ TEST_CASE("CheckCopyMovingNoisyLinearTest", "[FeedForwardNetworkTest]") arma::mat output = arma::mat("0"); // Check copying constructor. - FFN *model1 = new FFN(); - model1->ResetData(input, output); - model1->Add(5); - model1->Add(1); - model1->Add(); + FFN> *model1 = new FFN>(); + model1->Predictors() = input; + model1->Responses() = output; + model1->Add>(); + model1->Add>(10, 5); + model1->Add >(5, 1); + model1->Add>(); // Check whether copy constructor is working or not. - CheckCopyFunction(model1, input, output); + CheckCopyFunction<>(model1, input, output, 1); // Check moving constructor. - FFN *model2 = new FFN(); - model2->ResetData(input, output); - model2->Add(5); - model2->Add(1); - model2->Add(); + FFN> *model2 = new FFN>(); + model2->Predictors() = input; + model2->Responses() = output; + model2->Add>(); + model2->Add>(10, 5); + model2->Add >(5, 1); + model2->Add>(); // Check whether move constructor is working or not. - CheckMoveFunction(model2, input, output, 1); + CheckMoveFunction<>(model2, input, output, 1); } /** @@ -301,39 +291,43 @@ TEST_CASE("CheckCopyMovingConcatenateTest", "[FeedForwardNetworkTest]") arma::mat output = arma::mat("1"); // Check copying constructor. - FFN *model1 = new FFN(); - model1->ResetData(input, output); - model1->Add(5); + FFN> *model1 = new FFN>(); + model1->Predictors() = input; + model1->Responses() = output; + model1->Add>(); + model1->Add>(10, 5); // Create concatenate layer. arma::mat concatMatrix = arma::ones(5, 1); - Concatenate* concatLayer = new Concatenate(); + Concatenate<>* concatLayer = new Concatenate<>(); concatLayer->Concat() = concatMatrix; // Add concatenate layer to the current network. model1->Add(concatLayer); - model1->Add(5); - model1->Add(); + model1->Add >(10, 5); + model1->Add>(); // Check whether copy constructor is working or not. - CheckCopyFunction(model1, input, output); + CheckCopyFunction<>(model1, input, output, 1); // Check moving constructor. - FFN *model2 = new FFN(); - model2->ResetData(input, output); - model2->Add(5); + FFN> *model2 = new FFN>(); + model2->Predictors() = input; + model2->Responses() = output; + model2->Add>(); + model2->Add>(10, 5); // Create new concat layer. - Concatenate* concatLayer2 = new Concatenate(); + Concatenate<>* concatLayer2 = new Concatenate<>(); concatLayer2->Concat() = concatMatrix; // Add concatenate layer to the current network. model2->Add(concatLayer2); - model2->Add(5); - model2->Add(); + model2->Add >(10, 5); + model2->Add>(); // Check whether move constructor is working or not. - CheckMoveFunction(model2, input, output, 1); + CheckMoveFunction<>(model2, input, output, 1); } /** @@ -349,25 +343,47 @@ TEST_CASE("CheckCopyMovingDropoutNetworkTest", "[FeedForwardNetworkTest]") arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; trainData.shed_row(trainData.n_rows - 1); - FFN *model = new FFN; - model->Add(8); - model->Add(); - model->Add(0.3); - model->Add(3); - model->Add(); + /* + * Construct a feed forward network with trainData.n_rows input nodes, + * hiddenLayerSize hidden nodes and trainLabels.n_rows output nodes. The + * network structure looks like: + * + * Input Hidden Output + * Layer Layer Layer + * +-----+ +-----+ +-----+ + * | | | | | | + * | +------>| +------>| | + * | | +>| | +>| | + * +-----+ | +--+--+ | +-----+ + * | | + * Bias | Bias | + * Layer | Layer | + * +-----+ | +-----+ | + * | | | | | | + * | +-----+ | +-----+ + * | | | | + * +-----+ +-----+ + */ - FFN *model1 = new FFN; - model1->Add(8); - model1->Add(); - model1->Add(0.3); - model1->Add(3); - model1->Add(); + FFN > *model = new FFN >; + model->Add >(trainData.n_rows, 8); + model->Add >(); + model->Add >(0.3); + model->Add >(8, 3); + model->Add >(); + + FFN > *model1 = new FFN >; + model1->Add >(trainData.n_rows, 8); + model1->Add >(); + model1->Add >(0.3); + model1->Add >(8, 3); + model1->Add >(); // Check whether copy constructor is working or not. - CheckCopyFunction(model, trainData, trainLabels); + CheckCopyFunction<>(model, trainData, trainLabels, 1); // Check whether move constructor is working or not. - CheckMoveFunction(model1, trainData, trainLabels, 1); + CheckMoveFunction<>(model1, trainData, trainLabels, 1); } /** @@ -398,20 +414,20 @@ TEST_CASE("CheckCopyMovingVanillaNetworkTestNoBias", "[FeedForwardNetworkTest]") * +-----+ +--+--+ +-----+ */ - FFN *model = new FFN; - model->Add(8); - model->Add(); - model->Add(3); - model->Add(); + FFN > *model = new FFN >; + model->Add >(trainData.n_rows, 8); + model->Add >(); + model->Add >(8, 3); + model->Add >(); - FFN *model1 = new FFN; - model1->Add(8); - model1->Add(); - model1->Add(3); - model1->Add(); + FFN > *model1 = new FFN >; + model1->Add >(trainData.n_rows, 8); + model1->Add >(); + model1->Add >(8, 3); + model1->Add >(); // Check whether copy constructor is working or not. - CheckCopyFunction<>(model, trainData, trainLabels); + CheckCopyFunction<>(model, trainData, trainLabels, 1); // Check whether move constructor is working or not. CheckMoveFunction<>(model1, trainData, trainLabels, 1); @@ -420,38 +436,38 @@ TEST_CASE("CheckCopyMovingVanillaNetworkTestNoBias", "[FeedForwardNetworkTest]") /** * Check whether copying and moving network with Reparametrization is working or not. */ -// TEST_CASE("CheckCopyMovingReparametrizationNetworkTestNoBias", -// "[FeedForwardNetworkTest]") -// { -// // Load the dataset. -// arma::mat trainData; -// data::Load("thyroid_train.csv", trainData, true); -// -// // Normalize labels to [0, 2]. -// arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; -// trainData.shed_row(trainData.n_rows - 1); -// -// /* -// * Construct a feed forward network with trainData.n_rows input nodes, -// * followed by a linear layer and then a reparametrization layer. -// */ -// -// FFN *model = new FFN; -// model->Add >(trainData.n_rows, 8); -// model->Add >(4, false, true, 1); -// model->Add >(); -// -// FFN *model1 = new FFN; -// model1->Add >(trainData.n_rows, 8); -// model1->Add >(4, false, true, 1); -// model1->Add >(); -// -// // Check whether copy constructor is working or not. -// CheckCopyFunction<>(model, trainData, trainLabels, 1); -// -// // Check whether move constructor is working or not. -// CheckMoveFunction<>(model1, trainData, trainLabels, 1); -// } +TEST_CASE("CheckCopyMovingReparametrizationNetworkTestNoBias", + "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + data::Load("thyroid_train.csv", trainData, true); + + // Normalize labels to [0, 2]. + arma::mat trainLabels = trainData.row(trainData.n_rows - 1) - 1; + trainData.shed_row(trainData.n_rows - 1); + + /* + * Construct a feed forward network with trainData.n_rows input nodes, + * followed by a linear layer and then a reparametrization layer. + */ + + FFN > *model = new FFN >; + model->Add >(trainData.n_rows, 8); + model->Add >(4, false, true, 1); + model->Add >(); + + FFN > *model1 = new FFN >; + model1->Add >(trainData.n_rows, 8); + model1->Add >(4, false, true, 1); + model1->Add >(); + + // Check whether copy constructor is working or not. + CheckCopyFunction<>(model, trainData, trainLabels, 1); + + // Check whether move constructor is working or not. + CheckMoveFunction<>(model1, trainData, trainLabels, 1); +} /** * Train the vanilla network on a larger dataset. @@ -497,11 +513,11 @@ TEST_CASE("FFVanillaNetworkTest", "[FeedForwardNetworkTest]") * +-----+ +-----+ */ - FFN model; - model.Add(8); - model.Add(); - model.Add(3); - model.Add(); + FFN > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(8, 3); + model.Add >(); // Vanilla neural net with logistic activation function. // Because 92% of the patients are not hyperthyroid the neural @@ -518,13 +534,13 @@ TEST_CASE("FFVanillaNetworkTest", "[FeedForwardNetworkTest]") arma::mat labels = arma::zeros(1, dataset.n_cols); labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); - FFN model1; - model1.Add(10); - model1.Add(); - model1.Add(2); - model1.Add(); + FFN > model1; + model1.Add >(dataset.n_rows, 10); + model1.Add >(); + model1.Add >(10, 2); + model1.Add >(); // Vanilla neural net with logistic activation function. - TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2); + TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2); } TEST_CASE("ForwardBackwardTest", "[FeedForwardNetworkTest]") @@ -539,13 +555,14 @@ TEST_CASE("ForwardBackwardTest", "[FeedForwardNetworkTest]") arma::mat labels = arma::zeros(1, dataset.n_cols); labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); - FFN model; - model.Add(50); - model.Add(); - model.Add(10); - model.Add(); + FFN > model; + model.Add >(dataset.n_rows, 50); + model.Add >(); + model.Add >(50, 10); + model.Add >(); ens::VanillaUpdate opt; + model.ResetParameters(); #if ENS_VERSION_MAJOR == 1 opt.Initialize(model.Parameters().n_rows, model.Parameters().n_cols); #else @@ -645,12 +662,12 @@ TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]") * +-----+ */ - FFN model; - model.Add(8); - model.Add(); - model.Add(); - model.Add(3); - model.Add(); + FFN > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(); + model.Add >(8, 3); + model.Add >(); // Vanilla neural net with logistic activation function. // Because 92% of the patients are not hyperthyroid the neural @@ -668,19 +685,19 @@ TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]") arma::mat labels = arma::zeros(1, dataset.n_cols); labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); - FFN model1; - model1.Add(10); - model1.Add(); - model.Add(); - model1.Add(2); - model1.Add(); + FFN > model1; + model1.Add >(dataset.n_rows, 10); + model1.Add >(); + model.Add >(); + model1.Add >(10, 2); + model1.Add >(); // Vanilla neural net with logistic activation function. - TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2); + TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2); } /** * Train the highway network on a larger dataset. - * + */ TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]") { arma::mat dataset; @@ -693,16 +710,16 @@ TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]") arma::mat labels = arma::zeros(1, dataset.n_cols); labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); - FFN model; - model.Add(10); - Highway* highway = new Highway(); - highway->Add(10); - highway->Add(); + FFN > model; + model.Add >(dataset.n_rows, 10); + Highway<>* highway = new Highway<>(10, true); + highway->Add >(10, 10); + highway->Add >(); model.Add(highway); // This takes ownership of the memory. - model.Add(2); - model.Add(); - TestNetwork(model, dataset, labels, dataset, labels, 10, 0.2); -}*/ + model.Add >(10, 2); + model.Add >(); + TestNetwork<>(model, dataset, labels, dataset, labels, 10, 0.2); +} /** * Train the DropConnect network on a larger dataset. @@ -750,16 +767,16 @@ TEST_CASE("DropConnectNetworkTest", "[FeedForwardNetworkTest]") * */ - FFN model; - model.Add(8); - model.Add(); - model.Add(3); - model.Add(); + FFN > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(8, 3); + model.Add >(); // Vanilla neural net with logistic activation function. // Because 92% of the patients are not hyperthyroid the neural // network must be significant better than 92%. - TestNetwork(model, trainData, trainLabels, testData, testLabels, 10, 0.1); + TestNetwork<>(model, trainData, trainLabels, testData, testLabels, 10, 0.1); arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -771,14 +788,13 @@ TEST_CASE("DropConnectNetworkTest", "[FeedForwardNetworkTest]") arma::mat labels = arma::zeros(1, dataset.n_cols); labels.submat(0, labels.n_cols / 2, 0, labels.n_cols - 1).fill(1); - FFN model1; - model1.Add(10); - model1.Add(); - model1.Add(2); - model1.Add(); - + FFN > model1; + model1.Add >(dataset.n_rows, 10); + model1.Add >(); + model1.Add >(10, 2); + model1.Add >(); // Vanilla neural net with logistic activation function. - TestNetwork(model1, dataset, labels, dataset, labels, 10, 0.2); + TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2); } /** @@ -787,9 +803,9 @@ TEST_CASE("DropConnectNetworkTest", "[FeedForwardNetworkTest]") */ TEST_CASE("FFNMiscTest", "[FeedForwardNetworkTest]") { - FFN model; - model.Add(3); - model.Add(); + FFN> model; + model.Add>(2, 3); + model.Add>(); auto copiedModel(model); copiedModel = model; @@ -822,19 +838,19 @@ TEST_CASE("FFSerializationTest", "[FeedForwardNetworkTest]") // Vanilla neural net with logistic activation function. // Because 92% of the patients are not hyperthyroid the neural // network must be significant better than 92%. - FFN model; - model.Add(8); - model.Add(); - model.Add(); - model.Add(3); - model.Add(); + FFN > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(); + model.Add >(8, 3); + model.Add >(); ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1); model.Train(trainData, trainLabels, opt); - FFN xmlModel, jsonModel, binaryModel; - xmlModel.Add(10); // Layer that will get removed. + FFN> xmlModel, jsonModel, binaryModel; + xmlModel.Add>(10, 10); // Layer that will get removed. // Serialize into other models. SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); @@ -843,7 +859,7 @@ TEST_CASE("FFSerializationTest", "[FeedForwardNetworkTest]") model.Predict(testData, predictions); xmlModel.Predict(testData, xmlPredictions); jsonModel.Predict(testData, jsonPredictions); - binaryModel.Predict(testData, binaryPredictions); + jsonModel.Predict(testData, binaryPredictions); CheckMatrices(predictions, xmlPredictions, jsonPredictions, binaryPredictions); @@ -852,112 +868,110 @@ TEST_CASE("FFSerializationTest", "[FeedForwardNetworkTest]") /** * Test that serialization works ok for PReLU. */ -// TEST_CASE("PReLUSerializationTest", "[FeedForwardNetworkTest]") -// { -// // Load the dataset. -// arma::mat trainData; -// if (!data::Load("thyroid_train.csv", trainData)) -// FAIL("Cannot open thyroid_train.csv"); -// -// arma::mat trainLabels = trainData.row(trainData.n_rows - 1); -// trainData.shed_row(trainData.n_rows - 1); -// trainLabels -= 1; // The labels should be between 0 and numClasses - 1. -// -// arma::mat testData; -// if (!data::Load("thyroid_test.csv", testData)) -// FAIL("Cannot load dataset thyroid_test.csv"); -// -// arma::mat testLabels = testData.row(testData.n_rows - 1); -// testData.shed_row(testData.n_rows - 1); -// testLabels -= 1; // The labels should be between 0 and numClasses - 1. -// -// // Vanilla neural net with logistic activation function. -// // Because 92% of the patients are not hyperthyroid the neural -// // network must be significant better than 92%. -// FFN model; -// model.Add >(trainData.n_rows, 8); -// model.Add >(); -// model.Add >(); -// model.Add >(8, 3); -// model.Add >(); -// -// ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1); -// -// model.Train(trainData, trainLabels, opt); -// -// FFN xmlModel, jsonModel, binaryModel; -// xmlModel.Add>(10, 10); // Layer that will get removed. -// -// // Serialize into other models. -// SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); -// -// arma::mat predictions, xmlPredictions, jsonPredictions, binaryPredictions; -// model.Predict(testData, predictions); -// xmlModel.Predict(testData, xmlPredictions); -// jsonModel.Predict(testData, jsonPredictions); -// jsonModel.Predict(testData, binaryPredictions); -// -// CheckMatrices(predictions, xmlPredictions, jsonPredictions, -// binaryPredictions); -// } +TEST_CASE("PReLUSerializationTest", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + if (!data::Load("thyroid_train.csv", trainData)) + FAIL("Cannot open thyroid_train.csv"); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + trainLabels -= 1; // The labels should be between 0 and numClasses - 1. + + arma::mat testData; + if (!data::Load("thyroid_test.csv", testData)) + FAIL("Cannot load dataset thyroid_test.csv"); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + testLabels -= 1; // The labels should be between 0 and numClasses - 1. + + // Vanilla neural net with logistic activation function. + // Because 92% of the patients are not hyperthyroid the neural + // network must be significant better than 92%. + FFN > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(); + model.Add >(8, 3); + model.Add >(); + + ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1); + + model.Train(trainData, trainLabels, opt); + + FFN> xmlModel, jsonModel, binaryModel; + xmlModel.Add>(10, 10); // Layer that will get removed. + + // Serialize into other models. + SerializeObjectAll(model, xmlModel, jsonModel, binaryModel); + + arma::mat predictions, xmlPredictions, jsonPredictions, binaryPredictions; + model.Predict(testData, predictions); + xmlModel.Predict(testData, xmlPredictions); + jsonModel.Predict(testData, jsonPredictions); + jsonModel.Predict(testData, binaryPredictions); + + CheckMatrices(predictions, xmlPredictions, jsonPredictions, + binaryPredictions); +} /** * Test if the custom layers work. The target is to see if the code compiles * when the Train and Prediction are called. */ -// TEST_CASE("CustomLayerTest", "[FeedForwardNetworkTest]") -// { -// // Load the dataset. -// arma::mat trainData; -// if (!data::Load("thyroid_train.csv", trainData)) -// FAIL("Cannot open thyroid_train.csv"); -// -// arma::mat trainLabels = trainData.row(trainData.n_rows - 1); -// trainData.shed_row(trainData.n_rows - 1); -// trainLabels -= 1; // The labels should be between 0 and numClasses - 1. -// -// arma::mat testData; -// if (!data::Load("thyroid_test.csv", testData)) -// FAIL("Cannot load dataset thyroid_test.csv"); -// -// arma::mat testLabels = testData.row(testData.n_rows - 1); -// testData.shed_row(testData.n_rows - 1); -// testLabels -= 1; // The labels should be between 0 and numClasses - 1. -// -// FFN > model; -// model.Add >(trainData.n_rows, 8); -// model.Add >(); -// model.Add >(8, 3); -// model.Add >(); -// -// ens::RMSProp opt(0.01, 32, 0.88, 1e-8, 15, -1); -// model.Train(trainData, trainLabels, opt); -// -// arma::mat predictionTemp; -// model.Predict(testData, predictionTemp); -// arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); -// } +TEST_CASE("CustomLayerTest", "[FeedForwardNetworkTest]") +{ + // Load the dataset. + arma::mat trainData; + if (!data::Load("thyroid_train.csv", trainData)) + FAIL("Cannot open thyroid_train.csv"); + + arma::mat trainLabels = trainData.row(trainData.n_rows - 1); + trainData.shed_row(trainData.n_rows - 1); + trainLabels -= 1; // The labels should be between 0 and numClasses - 1. + + arma::mat testData; + if (!data::Load("thyroid_test.csv", testData)) + FAIL("Cannot load dataset thyroid_test.csv"); + + arma::mat testLabels = testData.row(testData.n_rows - 1); + testData.shed_row(testData.n_rows - 1); + testLabels -= 1; // The labels should be between 0 and numClasses - 1. + + FFN, RandomInitialization, CustomLayer<> > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(8, 3); + model.Add >(); + + ens::RMSProp opt(0.01, 32, 0.88, 1e-8, 15, -1); + model.Train(trainData, trainLabels, opt); + + arma::mat predictionTemp; + model.Predict(testData, predictionTemp); + arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); +} /** * Test the overload of Forward function which allows partial forward pass. */ TEST_CASE("PartialForwardTest", "[FeedForwardNetworkTest]") { - FFN model; - model.Add(10); + FFN, RandomInitialization> model; + model.Add >(5, 10); - // Add a new Add<> module which adds a (learnable) constant term to the input. - Add* addModule = new Add(); + // Add a new Add<> module which adds a constant term to the input. + Add<>* addModule = new Add<>(10); model.Add(addModule); - LinearNoBias* linearNoBiasModule = new LinearNoBias(10); + LinearNoBias<>* linearNoBiasModule = new LinearNoBias<>(10, 10); model.Add(linearNoBiasModule); - model.Add(10); - - // Set up the network for inputs of dimensionality 10. - model.Reset(10); + model.Add >(10, 10); + model.ResetParameters(); // Set the parameters of the Add<> module to a matrix of ones. addModule->Parameters() = arma::ones(10, 1); // Set the parameters of the LinearNoBias<> module to a matrix of ones. @@ -1012,12 +1026,12 @@ TEST_CASE("FFNTrainReturnObjective", "[FeedForwardNetworkTest]") // Vanilla neural net with logistic activation function. // Because 92% of the patients are not hyperthyroid the neural // network must be significantly better than 92%. - FFN model; - model.Add(8); - model.Add(); - model.Add(); - model.Add(3); - model.Add(); + FFN > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(); + model.Add >(8, 3); + model.Add >(); ens::RMSProp opt(0.01, 32, 0.88, 1e-8, trainData.n_cols /* 1 epoch */, -1); @@ -1032,14 +1046,14 @@ TEST_CASE("FFNTrainReturnObjective", "[FeedForwardNetworkTest]") TEST_CASE("FFNReturnModel", "[FeedForwardNetworkTest]") { // Create dummy network. - FFN model; - Linear* linearA = new Linear(3); + FFN > model; + Linear<>* linearA = new Linear<>(3, 3); model.Add(linearA); - Linear* linearB = new Linear(4); + Linear<>* linearB = new Linear<>(3, 4); model.Add(linearB); - // Initialize network parameters, with a new input size of 3. - model.Reset(3); + // Initialize network parameter. + model.ResetParameters(); // Set all network parameter to one. model.Parameters().ones(); @@ -1049,8 +1063,9 @@ TEST_CASE("FFNReturnModel", "[FeedForwardNetworkTest]") // Get the layer parameter from layer A and layer B and store them in // parameterA and parameterB. - const arma::mat parameterA = model.Network()[0]->Parameters(); - const arma::mat parameterB = model.Network()[1]->Parameters(); + arma::mat parameterA, parameterB; + boost::apply_visitor(ParametersVisitor(parameterA), model.Model()[0]); + boost::apply_visitor(ParametersVisitor(parameterB), model.Model()[1]); CheckMatrices(parameterA, arma::ones(3 * 3 + 3, 1)); CheckMatrices(parameterB, arma::zeros(3 * 4 + 4, 1)); @@ -1082,10 +1097,11 @@ TEST_CASE("OptimizerTest", "[FeedForwardNetworkTest]") testData.shed_row(testData.n_rows - 1); testLabels -= 1; // The labels should be between 0 and numClasses. - FFN model; - model.Add(8); - model.Add(3); - model.Add(); + FFN, RandomInitialization, CustomLayer<> > model; + model.Add >(trainData.n_rows, 8); + model.Add >(); + model.Add >(8, 3); + model.Add >(); ens::DE opt(200, 1000, 0.6, 0.8, 1e-5); model.Train(trainData, trainLabels, opt); @@ -1109,18 +1125,23 @@ TEST_CASE("FFNCheckInputShapeTest", "[FeedForwardNetworkTest]") arma::mat testData; data::Load("thyroid_test.csv", testData, true); - arma::mat testLabels = testData.row(testData.n_rows - 1); + arma::mat testLabels = testData.row(testData.n_rows - 1) - 1; testData.shed_row(testData.n_rows - 1); - FFN model; - model.Add(8); - model.Add(3); - model.Add(); + FFN, RandomInitialization, CustomLayer<> > model; + // Purposely putting wrong input shape so that error is thrown. + model.Add >(trainData.n_rows - 3, 8); + model.Add >(); + model.Add >(8, 3); + model.Add >(); + + std::string expectedMsg = "FFN<>::Train(): "; + expectedMsg += "the first layer of the network expects "; + expectedMsg += std::to_string(trainData.n_rows - 3) + " elements, "; + expectedMsg += "but the input has " + std::to_string(trainData.n_rows) + + " dimensions! "; ens::DE opt(200, 1000, 0.6, 0.8, 1e-5); - // Now set up the input incorrectly. - model.InputDimensions() = std::vector({ 1, 2, 3 }); - REQUIRE_THROWS_AS(model.Train(trainData, trainLabels, opt), std::logic_error); } diff --git a/src/mlpack/tests/not_adapted/gan_test.cpp b/src/mlpack/tests/gan_test.cpp similarity index 100% rename from src/mlpack/tests/not_adapted/gan_test.cpp rename to src/mlpack/tests/gan_test.cpp diff --git a/src/mlpack/tests/init_rules_test.cpp b/src/mlpack/tests/init_rules_test.cpp index 5d56e904b8..46c76e32f1 100644 --- a/src/mlpack/tests/init_rules_test.cpp +++ b/src/mlpack/tests/init_rules_test.cpp @@ -214,17 +214,18 @@ TEST_CASE("NetworkInitTest", "[InitRulesTest]") { arma::mat input = arma::ones(5, 1); arma::mat response; - NegativeLogLikelihood outputLayer; + NegativeLogLikelihood<> outputLayer; // Create a simple network and use the RandomInitialization rule to // initialize the network parameters. RandomInitialization randomInit(0.5, 0.5); - FFN randomModel( + FFN, RandomInitialization> randomModel( std::move(outputLayer), randomInit); - randomModel.Add(5); - randomModel.Add(2); - randomModel.Add(); + randomModel.Add >(); + randomModel.Add >(5, 5); + randomModel.Add >(5, 2); + randomModel.Add >(); randomModel.Predict(input, response); bool b = arma::all(arma::vectorise(randomModel.Parameters()) == 0.5); @@ -233,21 +234,23 @@ TEST_CASE("NetworkInitTest", "[InitRulesTest]") // Create a simple network and use the OrthogonalInitialization rule to // initialize the network parameters. - FFN orthogonalModel; - orthogonalModel.Add(5); - orthogonalModel.Add(2); - orthogonalModel.Add(); + FFN, OrthogonalInitialization> orthogonalModel; + orthogonalModel.Add >(); + orthogonalModel.Add >(5, 5); + orthogonalModel.Add >(5, 2); + orthogonalModel.Add >(); orthogonalModel.Predict(input, response); REQUIRE(orthogonalModel.Parameters().n_elem == 42); // Create a simple network and use the ZeroInitialization rule to // initialize the network parameters. - FFN - zeroModel(NegativeLogLikelihood(), ConstInitialization(0)); - zeroModel.Add(5); - zeroModel.Add(2); - zeroModel.Add(); + FFN, ConstInitialization> + zeroModel(NegativeLogLikelihood<>(), ConstInitialization(0)); + zeroModel.Add >(); + zeroModel.Add >(5, 5); + zeroModel.Add >(5, 2); + zeroModel.Add >(); zeroModel.Predict(input, response); REQUIRE(arma::accu(zeroModel.Parameters()) == 0); @@ -258,31 +261,34 @@ TEST_CASE("NetworkInitTest", "[InitRulesTest]") // parameters. KathirvalavakumarSubavathiInitialization kathirvalavakumarSubavathiInit( input, 1.5); - FFN + FFN, KathirvalavakumarSubavathiInitialization> ksModel(std::move(outputLayer), kathirvalavakumarSubavathiInit); - ksModel.Add(5); - ksModel.Add(2); - ksModel.Add(); + ksModel.Add >(); + ksModel.Add >(5, 5); + ksModel.Add >(5, 2); + ksModel.Add >(); ksModel.Predict(input, response); REQUIRE(ksModel.Parameters().n_elem == 42); // Create a simple network and use the OivsInitialization rule to // initialize the network parameters. - FFN > oivsModel; - oivsModel.Add(5); - oivsModel.Add(2); - oivsModel.Add(); + FFN, OivsInitialization<> > oivsModel; + oivsModel.Add >(); + oivsModel.Add >(5, 5); + oivsModel.Add >(5, 2); + oivsModel.Add >(); oivsModel.Predict(input, response); REQUIRE(oivsModel.Parameters().n_elem == 42); // Create a simple network and use the GaussianInitialization rule to // initialize the network parameters. - FFN gaussianModel; - gaussianModel.Add(5); - gaussianModel.Add(2); - gaussianModel.Add(); + FFN, GaussianInitialization> gaussianModel; + gaussianModel.Add >(); + gaussianModel.Add >(5, 5); + gaussianModel.Add >(5, 2); + gaussianModel.Add >(); gaussianModel.Predict(input, response); REQUIRE(gaussianModel.Parameters().n_elem == 42); diff --git a/src/mlpack/tests/ksinit_test.cpp b/src/mlpack/tests/ksinit_test.cpp index 15a9dec67c..930df617e4 100644 --- a/src/mlpack/tests/ksinit_test.cpp +++ b/src/mlpack/tests/ksinit_test.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include #include @@ -75,12 +75,12 @@ void BuildVanillaNetwork(MatType& trainData, // Cauchy’s Inequality Based on Sensitivity Analysis" paper. KathirvalavakumarSubavathiInitialization init(trainData, 4.59); - FFN - model(MeanSquaredError(), init); + FFN, KathirvalavakumarSubavathiInitialization> + model(MeanSquaredError<>(), init); - model.Add(hiddenLayerSize); - model.Add(); - model.Add(outputSize); + model.Add >(trainData.n_rows, hiddenLayerSize); + model.Add >(); + model.Add >(hiddenLayerSize, outputSize); ens::RMSProp opt(0.01, 1, 0.88, 1e-8, maxEpochs * trainData.n_cols, 1e-18); diff --git a/src/mlpack/tests/layer_names_test.cpp b/src/mlpack/tests/layer_names_test.cpp new file mode 100644 index 0000000000..9d94f0ff67 --- /dev/null +++ b/src/mlpack/tests/layer_names_test.cpp @@ -0,0 +1,160 @@ +/** + * @file tests/layer_names_test.cpp + * @author Sreenik Seal + * + * Tests for testing the string representation of + * layers in mlpack's ANN module. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include + +#include "catch.hpp" + +using namespace mlpack; +using namespace ann; + +/** + * Test if the LayerNameVisitor works properly. + */ +TEST_CASE("LayerNameVisitorTest", "[LayerNamesTest]") +{ + LayerTypes<> atrousConvolution = new AtrousConvolution<>(); + LayerTypes<> alphaDropout = new AlphaDropout<>(); + LayerTypes<> batchNorm = new BatchNorm<>(); + LayerTypes<> constant = new Constant<>(); + LayerTypes<> convolution = new Convolution<>(); + LayerTypes<> dropConnect = new DropConnect<>(); + LayerTypes<> dropout = new Dropout<>(); + LayerTypes<> flexibleReLU = new FlexibleReLU<>(); + LayerTypes<> layerNorm = new LayerNorm<>(); + LayerTypes<> linear = new Linear<>(); + LayerTypes<> linearNoBias = new LinearNoBias<>(); + LayerTypes<> maxPooling = new MaxPooling<>(); + LayerTypes<> meanPooling = new MeanPooling<>(); + LayerTypes<> multiplyConstant = new MultiplyConstant<>(); + LayerTypes<> reLULayer = new ReLULayer<>(); + LayerTypes<> transposedConvolution = new TransposedConvolution<>(); + LayerTypes<> identityLayer = new IdentityLayer<>(); + LayerTypes<> tanHLayer = new TanHLayer<>(); + LayerTypes<> eLU = new ELU<>(); + LayerTypes<> hardTanH = new HardTanH<>(); + LayerTypes<> leakyReLU = new LeakyReLU<>(); + LayerTypes<> pReLU = new PReLU<>(); + LayerTypes<> sigmoidLayer = new SigmoidLayer<>(); + LayerTypes<> logSoftMax = new LogSoftMax<>(); + LayerTypes<> lstmLayer = new LSTM<>(100, 10); + LayerTypes<> creluLayer = new CReLU<>(); + LayerTypes<> highwayLayer = new Highway<>(); + LayerTypes<> gruLayer = new GRU<>(); + LayerTypes<> glimpseLayer = new Glimpse<>(); + LayerTypes<> fastlstmLayer = new FastLSTM<>(); + LayerTypes<> weightnormLayer = new WeightNorm<>(new IdentityLayer<>()); + + // Bilinear interpolation is not yet supported by the string converter. + LayerTypes<> unsupportedLayer = new BilinearInterpolation<>(); + + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + atrousConvolution) == "atrousconvolution"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + alphaDropout) == "alphadropout"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + batchNorm) == "batchnorm"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + constant) == "constant"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + convolution) == "convolution"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + dropConnect) == "dropconnect"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + dropout) == "dropout"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + flexibleReLU) == "flexiblerelu"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + layerNorm) == "layernorm"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + linear) == "linear"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + linearNoBias) == "linearnobias"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + maxPooling) == "maxpooling"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + meanPooling) == "meanpooling"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + multiplyConstant) == "multiplyconstant"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + reLULayer) == "relu"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + transposedConvolution) == "transposedconvolution"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + identityLayer) == "identity"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + tanHLayer) == "tanh"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + eLU) == "elu"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + hardTanH) == "hardtanh"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + leakyReLU) == "leakyrelu"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + pReLU) == "prelu"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + sigmoidLayer) == "sigmoid"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + logSoftMax) == "logsoftmax"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + unsupportedLayer) == "unsupported"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + lstmLayer) == "lstm"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + creluLayer) == "crelu"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + highwayLayer) == "highway"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + gruLayer) == "gru"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + glimpseLayer) == "glimpse"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + fastlstmLayer) == "fastlstm"); + REQUIRE(boost::apply_visitor(LayerNameVisitor(), + weightnormLayer) == "weightnorm"); + // Delete all instances. + boost::apply_visitor(DeleteVisitor(), atrousConvolution); + boost::apply_visitor(DeleteVisitor(), alphaDropout); + boost::apply_visitor(DeleteVisitor(), batchNorm); + boost::apply_visitor(DeleteVisitor(), constant); + boost::apply_visitor(DeleteVisitor(), convolution); + boost::apply_visitor(DeleteVisitor(), dropConnect); + boost::apply_visitor(DeleteVisitor(), dropout); + boost::apply_visitor(DeleteVisitor(), flexibleReLU); + boost::apply_visitor(DeleteVisitor(), layerNorm); + boost::apply_visitor(DeleteVisitor(), linear); + boost::apply_visitor(DeleteVisitor(), linearNoBias); + boost::apply_visitor(DeleteVisitor(), maxPooling); + boost::apply_visitor(DeleteVisitor(), meanPooling); + boost::apply_visitor(DeleteVisitor(), multiplyConstant); + boost::apply_visitor(DeleteVisitor(), reLULayer); + boost::apply_visitor(DeleteVisitor(), transposedConvolution); + boost::apply_visitor(DeleteVisitor(), identityLayer); + boost::apply_visitor(DeleteVisitor(), tanHLayer); + boost::apply_visitor(DeleteVisitor(), eLU); + boost::apply_visitor(DeleteVisitor(), hardTanH); + boost::apply_visitor(DeleteVisitor(), leakyReLU); + boost::apply_visitor(DeleteVisitor(), pReLU); + boost::apply_visitor(DeleteVisitor(), sigmoidLayer); + boost::apply_visitor(DeleteVisitor(), logSoftMax); + boost::apply_visitor(DeleteVisitor(), unsupportedLayer); + boost::apply_visitor(DeleteVisitor(), lstmLayer); + boost::apply_visitor(DeleteVisitor(), creluLayer); + boost::apply_visitor(DeleteVisitor(), highwayLayer); + boost::apply_visitor(DeleteVisitor(), gruLayer); + boost::apply_visitor(DeleteVisitor(), glimpseLayer); + boost::apply_visitor(DeleteVisitor(), fastlstmLayer); + boost::apply_visitor(DeleteVisitor(), weightnormLayer); +} diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index f7699b4fc9..2c800b1e21 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -14,7 +14,7 @@ */ #include -#include +#include #include #include #include @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -55,7 +54,7 @@ TEST_CASE("HuberLossTest", "[LossFunctionsTest]") arma::mat input, target, output; arma::mat expectedOutput; double loss; - HuberLoss module; + HuberLoss<> module; // Test for sum reduction. input = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 " @@ -109,10 +108,10 @@ TEST_CASE("PoissonNLLLossTest", "[LossFunctionsTest]") arma::mat input, target, input4, target4; arma::mat output1, output2, output3, output4; arma::mat expOutput1, expOutput2, expOutput3, expOutput4; - PoissonNLLLoss module1(true, false, 1e-8, false); - PoissonNLLLoss module2(true, true, 1e-08, true); - PoissonNLLLoss module3(true, true, 1e-08, false); - PoissonNLLLoss module4(false, true, 1e-08, false); + PoissonNLLLoss<> module1(true, false, 1e-8, false); + PoissonNLLLoss<> module2(true, true, 1e-08, true); + PoissonNLLLoss<> module3(true, true, 1e-08, false); + PoissonNLLLoss<> module4(false, true, 1e-08, false); // Test the Forward function on a user generated input. input = arma::mat("1.0 1.0 1.9 1.6 -1.9 3.7 -1.0 0.5"); @@ -176,7 +175,7 @@ TEST_CASE("SimpleKLDivergenceTest", "[LossFunctionsTest]") arma::mat input, target, output; arma::mat expectedOutput; double loss; - KLDivergence module; + KLDivergence<> module; // Test for sum reduction. input = arma::mat("-0.7007 -2.0247 -0.7132 -0.4584 -0.2637 -1.1795 -0.1093 " @@ -225,9 +224,9 @@ TEST_CASE("SimpleKLDivergenceTest", "[LossFunctionsTest]") */ TEST_CASE("SimpleMeanSquaredLogarithmicErrorTest", "[LossFunctionsTest]") { - arma::mat input, target, output, expectedOutput; + arma::mat input, target, output, expectedOutput; double loss; - MeanSquaredLogarithmicError module; + MeanSquaredLogarithmicError<> module; // Test for sum reduction. input = arma::mat("-0.0494 1.1958 1.0486 -0.2121 1.6028 0.0737 -0.7091 " @@ -246,22 +245,28 @@ TEST_CASE("SimpleMeanSquaredLogarithmicErrorTest", "[LossFunctionsTest]") // Test the Backward function. module.Backward(input, target, output); - REQUIRE(arma::as_scalar(arma::accu(output)) == - Approx(-10.5619).epsilon(1e-3)); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-10.5619).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); CheckMatrices(output, expectedOutput, 0.1); - // Test the error function on a single input. - input = arma::mat("2"); - target = arma::mat("3"); - loss = module.Forward(input, target); - REQUIRE(loss == Approx(0.082760974810151655).epsilon(1e-3)); + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + expectedOutput = arma::mat("-0.0718 0.0585 0.1067 -0.2119 0.0348 -0.0907 " + "-0.9612 0.0482 0.0120 0.2070 0.0148 -0.0266"); + expectedOutput.reshape(4, 3); - // Test the Backward function on a single input. + // Test the Forward function. Loss should be 1.10606. + loss = module.Forward(input, target); + REQUIRE(loss == Approx(1.10606).epsilon(1e-3)); + + // Test the Backward function. module.Backward(input, target, output); - REQUIRE(arma::accu(output) == Approx(-0.1917880483011872).epsilon(1e-3)); - REQUIRE(output.n_elem == 1); + REQUIRE(arma::as_scalar(arma::accu(output)) == + Approx(-0.880156).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /* @@ -270,7 +275,7 @@ TEST_CASE("SimpleMeanSquaredLogarithmicErrorTest", "[LossFunctionsTest]") TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]") { arma::mat input, output, target; - MeanSquaredError module(false); + MeanSquaredError<> module(false); // Test the Forward function on a user generated input and compare it against // the manually calculated result. @@ -320,8 +325,8 @@ TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]") TEST_CASE("SimpleBinaryCrossEntropyLossTest", "[LossFunctionsTest]") { arma::mat input1, input2, input3, output, target1, target2, target3; - BCELoss module1(1e-6, true); - BCELoss module2(1e-6, false); + BCELoss<> module1(1e-6, true); + BCELoss<> module2(1e-6, false); // Test the Forward function on a user generator input and compare it against // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); @@ -373,7 +378,7 @@ TEST_CASE("SimpleSigmoidCrossEntropyErrorTest", "[LossFunctionsTest]") { arma::mat input1, input2, input3, output, target1, target2, target3, expectedOutput; - SigmoidCrossEntropyError module; + SigmoidCrossEntropyError<> module; // Test the Forward function on a user generator input and compare it against // the calculated result. @@ -434,7 +439,7 @@ TEST_CASE("SimpleEarthMoverDistanceLayerTest", "[LossFunctionsTest]") arma::mat input1, input2, output, target1, target2, expectedOutput; arma::mat input3, target3; double loss; - EarthMoverDistance module; + EarthMoverDistance<> module; // Test the Forward function on a user generator input and compare it against // the manually calculated result. @@ -504,10 +509,12 @@ TEST_CASE("GradientMeanSquaredErrorTest", "[LossFunctionsTest]") input = arma::randu(10, 1); target = arma::randu(2, 1); - model = new FFN(); - model->ResetData(input, target); - model->Add(2); - model->Add(); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 2); + model->Add >(); } ~GradientFunction() @@ -525,7 +532,7 @@ TEST_CASE("GradientMeanSquaredErrorTest", "[LossFunctionsTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, NguyenWidrowInitialization>* model; arma::mat input, target; } function; @@ -545,10 +552,12 @@ TEST_CASE("GradientReconstructionLossTest", "[LossFunctionsTest]") input = arma::randu(10, 1); target = arma::randu(2, 1); - model = new FFN(); - model->ResetData(input, target); - model->Add(2); - model->Add(); + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 2); + model->Add >(); } ~GradientFunction() @@ -566,7 +575,7 @@ TEST_CASE("GradientReconstructionLossTest", "[LossFunctionsTest]") arma::mat& Parameters() { return model->Parameters(); } - FFN* model; + FFN, NguyenWidrowInitialization>* model; arma::mat input, target; } function; @@ -580,7 +589,7 @@ TEST_CASE("DiceLossTest", "[LossFunctionsTest]") { arma::mat input1, input2, target, output; double loss; - DiceLoss module; + DiceLoss<> module; // Test the Forward function. Loss should be 0 if input = target. input1 = arma::ones(10, 1); @@ -621,7 +630,7 @@ TEST_CASE("SimpleMeanBiasErrorTest", "[LossFunctionsTest]") { arma::mat input, target, output; double loss; - MeanBiasError module; + MeanBiasError<> module; // Test for sum reduction. input = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 " @@ -671,7 +680,7 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]") { arma::mat input, target, output; double loss; - LogCoshLoss module(2); + LogCoshLoss<> module(2); // Test the Forward function. Loss should be 0 if input = target. input = arma::ones(10, 1); @@ -724,7 +733,7 @@ TEST_CASE("HingeEmbeddingLossTest", "[LossFunctionsTest]") { arma::mat input, target, output, expectedOutput; double loss; - HingeEmbeddingLoss module; + HingeEmbeddingLoss<> module; // Test for sum reduction input = arma::mat("0.1778 0.0957 0.1397 0.2256 0.1203 0.2403 0.1925 0.3144 " @@ -776,7 +785,7 @@ TEST_CASE("SimpleL1LossTest", "[LossFunctionsTest]") { arma::mat input, output, target; double loss; - L1Loss module(true); + L1Loss<> module(true); // Test the Forward function on a user generator input and compare it against // the manually calculated result. @@ -802,7 +811,7 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]") { arma::mat input1, input2, y, output; double loss; - CosineEmbeddingLoss module; + CosineEmbeddingLoss<> module; // Test the Forward function. Loss should be 0 if input1 = input2 and y = 1. input1 = arma::mat(1, 10); @@ -842,6 +851,44 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]") // Test the Backward function. module.Backward(input1, input2, output); REQUIRE(arma::accu(output) == Approx(0.06324556).epsilon(1e-3)); + + // Check for correctness for cube. + CosineEmbeddingLoss<> module2(0.5, true); + + arma::cube input3(3, 2, 2); + arma::cube input4(3, 2, 2); + input3.fill(1); + input4.fill(1); + input3(0) = 2; + input3(1) = 2; + input3(4) = 2; + input3(6) = 2; + input3(8) = 2; + input3(10) = 2; + input4(2) = 2; + input4(9) = 2; + input4(11) = 2; + loss = module2.Forward(input3, input4); + // Calculated using torch.nn.CosineEmbeddingLoss(). + REQUIRE(loss == Approx(0.55395).epsilon(1e-3)); + + // Test the Backward function. + module2.Backward(input3, input4, output); + REQUIRE(arma::accu(output) == Approx(-0.36649111).epsilon(1e-3)); + + // Check Output for mean type of reduction. + CosineEmbeddingLoss<> module3(0.0, true, false); + loss = module3.Forward(input3, input4); + REQUIRE(loss == Approx(0.092325).epsilon(1e-3)); + + // Check correctness for cube. + module3.Similarity() = false; + loss = module3.Forward(input3, input4); + REQUIRE(loss == Approx(0.90767498236).epsilon(1e-3)); + + // Test the Backward function. + module3.Backward(input3, input4, output); + REQUIRE(arma::accu(output) == Approx(0.0236749374).epsilon(1e-4)); } /* @@ -852,7 +899,7 @@ TEST_CASE("MarginRankingLossTest", "[LossFunctionsTest]") arma::mat input, input1, input2, target, output, expectedOutput; double loss; // Test sum reduction - MarginRankingLoss module; + MarginRankingLoss<> module; input1 = arma::mat("0.4287 -1.6208 -1.5006 -0.4473 1.5208 -4.5184 9.3574 " "-4.8090 4.3455 5.2070"); input2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 " @@ -878,7 +925,7 @@ TEST_CASE("MarginRankingLossTest", "[LossFunctionsTest]") // Test for mean reduction by modifying reduction parameter using accessor. module.Reduction() = false; - + // Test the forward function // loss should be 3.0353 // value calculated using torch.nn.MarginRankingLoss(margin=1.0,reduction='mean') @@ -903,8 +950,8 @@ TEST_CASE("SoftMarginLossTest", "[LossFunctionsTest]") { arma::mat input, target, output, expectedOutput; double loss; - SoftMarginLoss module1; - SoftMarginLoss module2(false); + SoftMarginLoss<> module1; + SoftMarginLoss<> module2(false); input = arma::mat("0.1778 0.0957 0.1397 0.1203 0.2403 0.1925 -0.2264 -0.3400 " "-0.3336"); @@ -958,7 +1005,7 @@ TEST_CASE("SoftMarginLossTest", "[LossFunctionsTest]") TEST_CASE("MeanAbsolutePercentageErrorTest", "[LossFunctionsTest]") { arma::mat input, target, output, expectedOutput; - MeanAbsolutePercentageError module; + MeanAbsolutePercentageError<> module; input = arma::mat("3 -0.5 2 7"); target = arma::mat("2.5 0.2 2 8"); @@ -978,20 +1025,6 @@ TEST_CASE("MeanAbsolutePercentageErrorTest", "[LossFunctionsTest]") CheckMatrices(output, expectedOutput, 0.1); } -/** - * Test that the function that can access the parameters of the - * VR Class Reward layer works. - */ -TEST_CASE("VRClassRewardLayerParametersTest", "[LossFunctionsTest]") -{ - // Parameter order : scale, sizeAverage. - VRClassReward layer(2, false); - - // Make sure we can get the parameters successfully. - REQUIRE(layer.Scale() == 2); - REQUIRE(layer.SizeAverage() == false); -} - /* * Simple test for the Triplet Margin Loss function. */ @@ -999,7 +1032,7 @@ TEST_CASE("TripletMarginLossTest") { arma::mat anchor, positive, negative; arma::mat input, target, output; - TripletMarginLoss module; + TripletMarginLoss<> module; // Test the Forward function on a user generated input and compare it against // the manually calculated result. @@ -1047,8 +1080,8 @@ TEST_CASE("HingeLossTest", "[LossFunctionsTest]") { arma::mat input, target, target_b, output; double loss, loss_b; - HingeLoss module1; - HingeLoss module2(false); + HingeLoss<> module1; + HingeLoss<> module2(false); // Test the Forward function. Loss should be 0 if input = target. input = arma::ones(10, 1); @@ -1124,8 +1157,8 @@ TEST_CASE("MultiLabelSoftMarginLossTest", "[LossFunctionsTest]") { arma::mat input, target, output, expectedOutput; double loss; - MultiLabelSoftMarginLoss module1; - MultiLabelSoftMarginLoss module2(false); + MultiLabelSoftMarginLoss<> module1; + MultiLabelSoftMarginLoss<> module2(false); input = arma::mat("0.1778 0.0957 0.1397 0.1203 0.2403 0.1925 -0.2264 -0.3400 " "-0.3336"); @@ -1183,8 +1216,8 @@ TEST_CASE("MultiLabelSoftMarginLossWeightedTest", "[LossFunctionsTest]") arma::rowvec weights; double loss; weights = arma::mat("1 2 3"); - MultiLabelSoftMarginLoss module1(true, weights); - MultiLabelSoftMarginLoss module2(false, weights); + MultiLabelSoftMarginLoss<> module1(true, weights); + MultiLabelSoftMarginLoss<> module2(false, weights); input = arma::mat("0.1778 0.0957 0.1397 0.2256 0.1203 0.2403 0.1925 0.3144 " "-0.2264 -0.3400 -0.3336 -0.8695"); @@ -1241,7 +1274,7 @@ TEST_CASE("NegativeLogLikelihoodLossTest", "[LossFunctionsTest]") arma::mat input, target, output; arma::mat expectedOutput; double loss; - NegativeLogLikelihood module; + NegativeLogLikelihood<> module; // Test for sum reduction. input = arma::mat("-0.1689 -2.0033 -3.8886 -0.2862 -1.9392 -2.2532" diff --git a/src/mlpack/tests/perceptron_test.cpp b/src/mlpack/tests/perceptron_test.cpp index 57422961fc..d53903c259 100644 --- a/src/mlpack/tests/perceptron_test.cpp +++ b/src/mlpack/tests/perceptron_test.cpp @@ -120,7 +120,7 @@ TEST_CASE("And", "[PerceptronTest]") mat testData; testData = { { 0, 1, 1, 0 }, { 1, 0, 1, 0 } }; - Row predictedLabels; + Row predictedLabels(testData.n_cols); p.Classify(testData, predictedLabels); CHECK(predictedLabels(0, 0) == 0); @@ -146,7 +146,7 @@ TEST_CASE("Or", "[PerceptronTest]") mat testData; testData = { { 0, 1, 1, 0 }, { 1, 0, 1, 0 } }; - Row predictedLabels; + Row predictedLabels(testData.n_cols); p.Classify(testData, predictedLabels); CHECK(predictedLabels(0, 0) == 1); @@ -173,7 +173,7 @@ TEST_CASE("Random3", "[PerceptronTest]") mat testData; testData = { { 0, 1, 1 }, { 1, 0, 1 } }; - Row predictedLabels; + Row predictedLabels(testData.n_cols); p.Classify(testData, predictedLabels); for (size_t i = 0; i < predictedLabels.n_cols; ++i) @@ -198,7 +198,7 @@ TEST_CASE("TwoPoints", "[PerceptronTest]") mat testData; testData = { { 0, 1 }, { 1, 0 } }; - Row predictedLabels; + Row predictedLabels(testData.n_cols); p.Classify(testData, predictedLabels); CHECK(predictedLabels(0, 0) == 0); @@ -223,7 +223,7 @@ TEST_CASE("NonLinearlySeparableDataset", "[PerceptronTest]") mat testData; testData = { { 3, 4, 5, 6 }, { 3, 2.3, 1.7, 1.5 } }; - Row predictedLabels; + Row predictedLabels(testData.n_cols); p.Classify(testData, predictedLabels); CHECK(predictedLabels(0, 0) == 0); @@ -244,28 +244,4 @@ TEST_CASE("SecondaryConstructor", "[PerceptronTest]") Perceptron<> p1(trainData, labels.row(0), 2, 1000); Perceptron<> p2(p1); - - REQUIRE(p1.Weights().n_elem > 0); - REQUIRE(p2.Weights().n_elem > 0); -} - -/** - * This tests that we can build the Perceptron when specifying instance weights. - */ -TEST_CASE("InstanceWeightsConstructor", "[PerceptronTest]") -{ - mat trainData; - trainData = { { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }, - { 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 } }; - - Mat labels; - labels = { 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1 }; - - rowvec instanceWeights; - instanceWeights = { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.9, - 0.8, 0.7, 0.6, 0.5, 0.4 }; - - Perceptron<> p(trainData, labels.row(0), 2, instanceWeights, 1000); - - REQUIRE(p.Weights().n_elem > 0); } diff --git a/src/mlpack/tests/not_adapted/rbm_network_test.cpp b/src/mlpack/tests/rbm_network_test.cpp similarity index 100% rename from src/mlpack/tests/not_adapted/rbm_network_test.cpp rename to src/mlpack/tests/rbm_network_test.cpp diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index a7d3c459c3..4b5aca4afb 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -11,16 +11,17 @@ */ #include -#include -#include -#include -#include -#include - #include +#include +#include +#include +#include +#include +#include #include "catch.hpp" #include "serialization.hpp" +#include "custom_layer.hpp" using namespace mlpack; using namespace mlpack::ann; @@ -68,32 +69,6 @@ void GenerateNoisySines(arma::cube& data, } } -/** - * Construct dataset for sine wave prediction. - * - * @param data Input data used to store the noisy sines. - * @param labels Labels used to store the target class of the noisy sines. - * @param points Number of points/features in a single sequence. - * @param sequences Number of sequences for each class. - * @param noise The noise factor that influences the sines. - */ -void GenerateSines(arma::cube& data, - arma::cube& labels, - const size_t sequences, - const size_t len) -{ - arma::vec x = arma::sin(arma::linspace(0, - sequences + len, sequences + len)); - data.set_size(1, len, sequences); - labels.set_size(1, 1, sequences); - - for (size_t i = 0; i < sequences; ++i) - { - data.slice(i) = arma::reshape(x.subvec(i, i + len), 1, len); - labels.slice(i) = x(i + len); - } -} - /* * This sample is a simplified version of Derek D. Monner's Distracted Sequence * Recall task, which involves 10 symbols: @@ -149,33 +124,150 @@ void GenerateDistractedSequence(arma::mat& input, arma::mat& output) output.reshape(output.n_elem, 1); } +/** + * Train the specified network and the construct distracted sequence recall + * dataset. + */ +template +void DistractedSequenceRecallTestNetwork( + const size_t cellSize, const size_t hiddenSize) +{ + const size_t trainDistractedSequenceCount = 600; + const size_t testDistractedSequenceCount = 300; + + arma::field trainInput(1, trainDistractedSequenceCount); + arma::field trainLabels(1, trainDistractedSequenceCount); + arma::field testInput(1, testDistractedSequenceCount); + arma::field testLabels(1, testDistractedSequenceCount); + + // Generate the training data. + for (size_t i = 0; i < trainDistractedSequenceCount; ++i) + GenerateDistractedSequence(trainInput(0, i), trainLabels(0, i)); + + // Generate the test data. + for (size_t i = 0; i < testDistractedSequenceCount; ++i) + GenerateDistractedSequence(testInput(0, i), testLabels(0, i)); + + /* + * Construct a network with 10 input units, layerSize hidden units and 3 + * output units. The hidden layer is connected to itself. The network + * structure looks like: + * + * Input Recurrent Hidden Output + * Layer(10) Layer(cellSize) Layer(3) Layer(3) + * +-----+ +-----+ +-----+ +-----+ + * | | | | | | | | + * | +------>| +------>| |------>| | + * | | ..>| | | | | | + * +-----+ . +--+--+ +-----+ +-----+ + * . . + * . . + * ....... + */ + const size_t outputSize = 3; + const size_t inputSize = 10; + const size_t rho = trainInput.at(0, 0).n_elem / inputSize; + + // It isn't guaranteed that the recurrent network will converge in the + // specified number of iterations using random weights. If this works 1 of 5 + // times, I'm fine with that. All I want to know is that the network is able + // to escape from local minima and to solve the task. + size_t successes = 0; + size_t offset = 0; + for (size_t trial = 0; trial < 5; ++trial) + { + RNN > model(rho); + model.Add >(); + model.Add >(inputSize, cellSize); + model.Add(cellSize, hiddenSize); + model.Add >(hiddenSize, outputSize); + model.Add >(); + + StandardSGD opt(0.1, 50, 2, -50000); + + // We increase the number of iterations (training) if the first run didn't + // pass. + arma::cube inputTemp, labelsTemp; + for (size_t iteration = 0; iteration < (9 + offset); iteration++) + { + for (size_t j = 0; j < trainDistractedSequenceCount; ++j) + { + inputTemp = arma::cube(trainInput.at(0, j).memptr(), inputSize, 1, + trainInput.at(0, j).n_elem / inputSize, false, true); + labelsTemp = arma::cube(trainLabels.at(0, j).memptr(), outputSize, 1, + trainLabels.at(0, j).n_elem / outputSize, false, true); + + model.Train(inputTemp, labelsTemp, opt); + } + } + + double error = 0; + + // Ask the network to predict the targets in the given sequence at the + // prompts. + for (size_t i = 0; i < testDistractedSequenceCount; ++i) + { + arma::cube output; + arma::cube input(testInput.at(0, i).memptr(), inputSize, 1, + testInput.at(0, i).n_elem / inputSize, false, true); + + model.Predict(input, output); + for (size_t j = 0; j < output.n_slices; ++j) + { + arma::mat outputSlice = output.slice(j); + data::Binarize(outputSlice, outputSlice, 0.5); + output.slice(j) = outputSlice; + } + + arma::cube label(testLabels.at(0, i).memptr(), outputSize, 1, + testLabels.at(0, i).n_elem / outputSize, false, true); + if (arma::accu(arma::abs(label - output)) != 0) + error += 1; + } + + error /= testDistractedSequenceCount; + // Can we reproduce the results from the paper. They provide an 95% accuracy + // on a test set of 1000 randomly selected sequences. + // Ensure that this is within tolerance, which is at least as good as the + // paper's results (plus a little bit for noise). + if (error <= 0.3) + { + ++successes; + break; + } + + offset += 2; + } + + REQUIRE(successes >= 1); +} /** * Train the specified networks on the Derek D. Monner's distracted sequence * recall task. */ -/* TEST_CASE("LSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") */ -/* { */ -/* DistractedSequenceRecallTestNetwork >(4, 8); */ -/* } */ +TEST_CASE("LSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") +{ + DistractedSequenceRecallTestNetwork >(4, 8); +} /** * Train the specified networks on the Derek D. Monner's distracted sequence * recall task. */ -/* TEST_CASE("FastLSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") */ -/* { */ -/* DistractedSequenceRecallTestNetwork >(4, 8); */ -/* } */ +TEST_CASE("FastLSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") +{ + DistractedSequenceRecallTestNetwork >(4, 8); +} /** * Train the specified networks on the Derek D. Monner's distracted sequence * recall task. */ -/* TEST_CASE("GRUDistractedSequenceRecallTest", "[RecurrentNetworkTest]") */ -/* { */ -/* DistractedSequenceRecallTestNetwork >(4, 8); */ -/* } */ +TEST_CASE("GRUDistractedSequenceRecallTest", "[RecurrentNetworkTest]") +{ + DistractedSequenceRecallTestNetwork >(4, 8); +} /** * Create a simple recurrent neural network for the noisy sines task, and @@ -184,16 +276,15 @@ void GenerateDistractedSequence(arma::mat& input, arma::mat& output) template void BatchSizeTest() { - const size_t T = 50; - const size_t bpttTruncate = 10; + const size_t rho = 10; // Generate 12 (2 * 6) noisy sines. A single sine contains rho // points/features. arma::cube input; arma::mat labelsTemp; - GenerateNoisySines(input, labelsTemp, 4, 5); + GenerateNoisySines(input, labelsTemp, rho, 6); - arma::cube labels = arma::zeros(1, labelsTemp.n_cols, T); + arma::cube labels = arma::zeros(1, labelsTemp.n_cols, rho); for (size_t i = 0; i < labelsTemp.n_cols; ++i) { const int value = arma::as_scalar(arma::find( @@ -201,15 +292,15 @@ void BatchSizeTest() labels.tube(0, i).fill(value); } - RNN<> model(bpttTruncate); - model.Add(100); - model.Add(); - model.Add(10); - model.Add(); - model.Add(10); - model.Add(); + RNN<> model(rho); + model.Add>(1, 10); + model.Add>(); + model.Add(10, 10); + model.Add>(); + model.Add>(10, 10); + model.Add>(); - model.Reset(1); + model.Reset(); arma::mat initParams = model.Parameters(); StandardSGD opt(1e-5, 1, 5, -100, false); @@ -218,14 +309,13 @@ void BatchSizeTest() // This is trained with one point. arma::mat outputParams = model.Parameters(); - model.Reset(1); + model.Reset(); model.Parameters() = initParams; opt.BatchSize() = 2; model.Train(input, labels, opt); CheckMatrices(outputParams, model.Parameters(), 1); - model.Reset(1); model.Parameters() = initParams; opt.BatchSize() = 5; model.Train(input, labels, opt); @@ -238,28 +328,28 @@ void BatchSizeTest() */ TEST_CASE("LSTMBatchSizeTest", "[RecurrentNetworkTest]") { - BatchSizeTest(); + BatchSizeTest>(); } /** * Ensure fast LSTMs work with larger batch sizes. */ -//TEST_CASE("FastLSTMBatchSizeTest", "[RecurrentNetworkTest]") -//{ -// BatchSizeTest>(); -//} +TEST_CASE("FastLSTMBatchSizeTest", "[RecurrentNetworkTest]") +{ + BatchSizeTest>(); +} /** * Ensure GRUs work with larger batch sizes. */ -//TEST_CASE("GRUBatchSizeTest", "[RecurrentNetworkTest]") -//{ -// BatchSizeTest>(); -//} +TEST_CASE("GRUBatchSizeTest", "[RecurrentNetworkTest]") +{ + BatchSizeTest>(); +} /** * Make sure the RNN can be properly serialized. - * + */ TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -293,7 +383,7 @@ TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") * . . * . . * ....... - * + */ Add<> add(4); Linear<> lookup(1, 4); SigmoidLayer<> sigmoidLayer; @@ -307,7 +397,7 @@ TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") model.Add >(4, 10); model.Add >(); - StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch *, -100); + StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); model.Train(input, labels, opt); // Serialize the network. @@ -323,11 +413,10 @@ TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") CheckMatrices(prediction, xmlPrediction, jsonPrediction, binaryPrediction); } -*/ /** * Train the BRNN on a larger dataset. - * + */ TEST_CASE("SequenceClassificationBRNNTest", "[RecurrentNetworkTest]") { // Using same test for RNN below. @@ -396,11 +485,10 @@ TEST_CASE("SequenceClassificationBRNNTest", "[RecurrentNetworkTest]") REQUIRE(successes >= 1); } -*/ /** * Train the vanilla network on a larger dataset. - * + */ TEST_CASE("SequenceClassificationTest", "[RecurrentNetworkTest]") { // It isn't guaranteed that the recurrent network will converge in the @@ -441,7 +529,7 @@ TEST_CASE("SequenceClassificationTest", "[RecurrentNetworkTest]") * . . * . . * ....... - * + */ Add<> add(4); Linear<> lookup(1, 4); SigmoidLayer<> sigmoidLayer; @@ -487,7 +575,6 @@ TEST_CASE("SequenceClassificationTest", "[RecurrentNetworkTest]") REQUIRE(successes >= 1); } -*/ /** * @brief Generates noisy sine wave and outputs the data and the labels that @@ -571,10 +658,12 @@ void GenerateNoisySinRNN(arma::cube& data, */ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100) { - RNN net(rho, true); - net.Add(hiddenUnits); - net.Add(hiddenUnits); - net.Add(1); + RNN > net(rho, true); + net.Add >(1, hiddenUnits); + net.Add >(hiddenUnits, hiddenUnits); + net.Add >(hiddenUnits, 1); + + RMSProp opt(0.005, 100, 0.9, 1e-08, 50000, 1e-5); // Generate data arma::cube data; @@ -589,12 +678,12 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100) arma::cube testLabels = labels.subcube(0, labels.n_cols - testCols, 0, labels.n_rows - 1, labels.n_cols - 1, labels.n_slices - 1); - RMSProp opt(0.005, 16, 0.9, 1e-08, trainCols * numEpochs, 1e-5); - - net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1, - data.n_slices - 1), labels.subcube(0, 0, 0, labels.n_rows - 1, - trainCols - 1, labels.n_slices - 1), opt); - + for (size_t i = 0; i < numEpochs; ++i) + { + net.Train(data.subcube(0, 0, 0, data.n_rows - 1, trainCols - 1, + data.n_slices - 1), labels.subcube(0, 0, 0, labels.n_rows - 1, + trainCols - 1, labels.n_slices - 1), opt); + } // Well now it should be trained. Do the test here. arma::cube prediction; net.Predict(testData, prediction); @@ -624,7 +713,7 @@ TEST_CASE("MultiTimestepTest", "[RecurrentNetworkTest]") /** * Test that RNN::Train() returns finite objective value. - * + */ TEST_CASE("RNNTrainReturnObjective", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -658,7 +747,7 @@ TEST_CASE("RNNTrainReturnObjective", "[RecurrentNetworkTest]") * . . * . . * ....... - * + */ Add<> add(4); Linear<> lookup(1, 4); SigmoidLayer<> sigmoidLayer; @@ -672,16 +761,15 @@ TEST_CASE("RNNTrainReturnObjective", "[RecurrentNetworkTest]") model.Add >(4, 10); model.Add >(); - StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch *, -100); + StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); double objVal = model.Train(input, labels, opt); REQUIRE(std::isfinite(objVal) == true); } -*/ /** * Test that BRNN::Train() returns finite objective value. - * + */ TEST_CASE("BRNNTrainReturnObjective", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -717,7 +805,6 @@ TEST_CASE("BRNNTrainReturnObjective", "[RecurrentNetworkTest]") // Test that BRNN::Train() returns finite objective value. REQUIRE(std::isfinite(objVal) == true); } -*/ /** * Test that RNN::Train() does not give an error for large rho. @@ -735,9 +822,10 @@ TEST_CASE("LargeRhoValueRnnTest", "[RecurrentNetworkTest]") RNN<> model(rho); - model.Add(hiddenSize); - model.Add(0.1); - model.Add(numLetters); + model.Add>(); + model.Add>(numLetters, hiddenSize, rho); + model.Add>(0.1); + model.Add>(hiddenSize, numLetters); const auto makeInput = [numLetters](const char *line) -> MatType { @@ -776,7 +864,7 @@ TEST_CASE("LargeRhoValueRnnTest", "[RecurrentNetworkTest]") inputs[i] = makeInput(trainingData[i].c_str()); targets[i] = makeTarget(trainingData[i].c_str()); } - ens::StandardSGD opt(0.01, 1, 100); + ens::SGD<> opt(0.01, 1, 100); model.Train(inputs[0], targets[0], opt); INFO("Training over"); } @@ -784,7 +872,7 @@ TEST_CASE("LargeRhoValueRnnTest", "[RecurrentNetworkTest]") /** * Test to make sure that an error is thrown when input with * wrong input shape is provided to a RNN. - * + */ TEST_CASE("RNNCheckInputShapeTest", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -818,7 +906,7 @@ TEST_CASE("RNNCheckInputShapeTest", "[RecurrentNetworkTest]") * . . * . . * ....... - * + */ Add<> add(4); // Purposely providing wrong input shape of 3. // The correct input shape is 1. @@ -839,41 +927,7 @@ TEST_CASE("RNNCheckInputShapeTest", "[RecurrentNetworkTest]") expectedMsg += std::to_string(3) + " elements, "; expectedMsg += "but the input has " + std::to_string(1) + " dimensions! "; - StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch *, -100); + StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); REQUIRE_THROWS_AS(model.Train(input, labels, opt), std::logic_error); } -*/ - -/** - * Test that a simple RNN with no recurrent components behaves the same as an - * FFN. - */ -TEST_CASE("RNNFFNTest", "[RecurrentNetworkTest]") -{ - // We'll create an RNN with *no* BPTT, just a simple single-layer linear - // network. - RNN rnn; - FFN ffn; - - rnn.Add(10); - rnn.Add(); - rnn.Add(1); - - ffn.Add(10); - ffn.Add(); - ffn.Add(1); - - // Now create some random data. - arma::cube data(20, 200, 1, arma::fill::randu); - arma::cube responses(1, 200, 1, arma::fill::randu); - - // Train the FFN. - ens::StandardSGD optimizer(1e-5, 1, 200, 1e-8, false); - - ffn.Train(data.slice(0), responses.slice(0), optimizer); - rnn.Train(data, responses, optimizer); - - // Now, the weights should be the same! - CheckMatrices(ffn.Parameters(), rnn.Parameters()); -} diff --git a/src/mlpack/tests/reward_clipping_test.cpp b/src/mlpack/tests/reward_clipping_test.cpp index fe2381d4b9..d1fd873740 100644 --- a/src/mlpack/tests/reward_clipping_test.cpp +++ b/src/mlpack/tests/reward_clipping_test.cpp @@ -22,7 +22,7 @@ #include #include -#include +#include #include #include #include @@ -62,7 +62,7 @@ TEST_CASE("RewardClippedAcrobotWithDQN", "[RewardClippingTest]") for (size_t trial = 0; trial < 3; ++trial) { // Set up the network. - SimpleDQN<> model(64, 32, 3); + SimpleDQN<> model(4, 64, 32, 3); // Set up the policy and replay method. GreedyPolicy> policy(1.0, 1000, 0.1, 0.99); diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index 9d4beb8b94..17b51cc865 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -38,7 +38,7 @@ #include #include #include -//#include +#include #include using namespace mlpack; @@ -1498,7 +1498,7 @@ TEST_CASE("HoeffdingTreeTest", "[SerializationTest]") /** * Build a Binary RBM, then save it and make sure the parameters of the * all the RBM are equal. - * + */ TEST_CASE("BinaryRBMTest", "[SerializationTest]") { arma::mat data; @@ -1531,12 +1531,11 @@ TEST_CASE("BinaryRBMTest", "[SerializationTest]") CheckMatrices(Rbm.Weight(), RbmText.Weight()); CheckMatrices(Rbm.Weight(), RbmBinary.Weight()); } -*/ /** * Build a ssRBM, then save it and make sure the parameters of the * all the RBM are equal. - * + */ TEST_CASE("ssRBMTest", "[SerializationTest]") { arma::mat data; @@ -1586,7 +1585,6 @@ TEST_CASE("ssRBMTest", "[SerializationTest]") CheckMatrices(Rbm.Weight(), RbmText.Weight()); CheckMatrices(Rbm.Weight(), RbmBinary.Weight()); } -*/ // Make sure serialization works for BayesianLinearRegression. TEST_CASE("BayesianLinearRegressionTest", "[SerializationTest]") diff --git a/src/mlpack/tests/not_adapted/wgan_test.cpp b/src/mlpack/tests/wgan_test.cpp similarity index 100% rename from src/mlpack/tests/not_adapted/wgan_test.cpp rename to src/mlpack/tests/wgan_test.cpp