From 7bd26a8ba20ad53a0bafe20b5603a8c18e1df133 Mon Sep 17 00:00:00 2001 From: Tarek Date: Sun, 18 Jun 2023 13:24:16 +0300 Subject: [PATCH 01/11] feat(rl): implement Ornstein-Uhlenbeck noise class Signed-off-by: Tarek --- .../reinforcement_learning/noise/noise.hpp | 17 ++++ .../noise/ornstein_uhlenbeck.hpp | 78 +++++++++++++++++++ .../reinforcement_learning.hpp | 1 + src/mlpack/tests/q_learning_test.cpp | 23 ++++++ 4 files changed, 119 insertions(+) create mode 100644 src/mlpack/methods/reinforcement_learning/noise/noise.hpp create mode 100644 src/mlpack/methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp diff --git a/src/mlpack/methods/reinforcement_learning/noise/noise.hpp b/src/mlpack/methods/reinforcement_learning/noise/noise.hpp new file mode 100644 index 0000000000..efa5998638 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/noise/noise.hpp @@ -0,0 +1,17 @@ +/** + * @file methods/reinforcement_learning/noise/noise.hpp + * @author Tarek Elsayed + * + * Convenience include for reinforcement learning noises. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_REINFORCEMENT_LEARNING_NOISE_NOISE_HPP +#define MLPACK_METHODS_REINFORCEMENT_LEARNING_NOISE_NOISE_HPP + +#include "ornstein_uhlenbeck.hpp" + +#endif diff --git a/src/mlpack/methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp b/src/mlpack/methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp new file mode 100644 index 0000000000..d2b6cf4ab8 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp @@ -0,0 +1,78 @@ +/** + * @file methods/reinforcement_learning/noise/ornstein_uhlenbeck.hpp + * @author Tarek Elsayed + * + * This file is the implementation of OUNoise class. + * Ornstein-Uhlenbeck process generates temporally correlated exploration, + * and it effectively copes with physical control problems of inertia. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_NOISE_ORNSTEIN_UHLENBECK_HPP +#define MLPACK_METHODS_RL_NOISE_ORNSTEIN_UHLENBECK_HPP + +#include + +namespace mlpack { +class OUNoise +{ + public: + /** + * @param size The size of the noise vector. + * @param mu The mean of the noise process. + * @param theta The rate of mean reversion. + * @param sigma The standard deviation of the noise. + */ + OUNoise(int size, + double mu = 0.0, + double theta = 0.15, + double sigma = 0.2) : + mu(mu * arma::ones(size)), + theta(theta), + sigma(sigma) + { + reset(); + } + + /** + * Reset the internal state to the mean (mu). + */ + void reset() + { + state = mu; + } + + /** + * Update the internal state and return it as a noise sample. + * + * @return Noise sample. + */ + arma::colvec sample() + { + arma::colvec x = state; + arma::colvec dx = theta * (mu - x) + + sigma * arma::randn(x.n_elem); + state = x + dx; + return state; + } + + private: + //! Locally-stored state of the noise process. + arma::colvec state; + + //! Locally-stored mean of the noise process. + arma::colvec mu; + + //! Locally-stored rate of mean reversion. + double theta; + + //! Locally-stored standard deviation of the noise. + double sigma; +}; + +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp b/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp index a9f37471d9..6af0276800 100644 --- a/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp @@ -17,6 +17,7 @@ #include "q_networks/q_networks.hpp" #include "replay/replay.hpp" #include "worker/worker.hpp" +#include "noise/noise.hpp" #include "training_config.hpp" #include "async_learning.hpp" diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index b48e7cd6bf..bdbc527e14 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -651,3 +651,26 @@ TEST_CASE("DDPGForMultipleActions", "[QLearningTest]") // If the agent is able to reach this point of the test, it is assured // that the agent can handle multiple actions in continuous space. } + +//! Test Ornstein-Uhlenbeck noise class. +TEST_CASE("OUNoiseTest", "[QLearningTest]") +{ + // Set up the OUNoise parameters. + int size = 3; + double mu = 0.0; + double theta = 0.15; + double sigma = 0.2; + + // Create an instance of the OUNoise class. + OUNoise ouNoise(size, mu, theta, sigma); + + // Test the reset function. + ouNoise.reset(); + arma::colvec state = ouNoise.sample(); + REQUIRE(state.n_elem == size); + + // Verify that the sample is not equal to the reset state. + arma::colvec sample = ouNoise.sample(); + bool isNotEqual = arma::any(sample != state); + REQUIRE(isNotEqual); +} From d8b7431bde34395de6cdf864819a755029361e7a Mon Sep 17 00:00:00 2001 From: Tarek Date: Tue, 20 Jun 2023 18:25:02 +0300 Subject: [PATCH 02/11] feat(rl): modify ddpg to accept a noise instance Signed-off-by: Tarek --- .../methods/reinforcement_learning/ddpg.hpp | 7 +++++ .../reinforcement_learning/ddpg_impl.hpp | 23 +++++++++++++--- src/mlpack/tests/q_learning_test.cpp | 27 ++++++++++++++++--- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/ddpg.hpp b/src/mlpack/methods/reinforcement_learning/ddpg.hpp index 03bf89b870..2bab884a71 100644 --- a/src/mlpack/methods/reinforcement_learning/ddpg.hpp +++ b/src/mlpack/methods/reinforcement_learning/ddpg.hpp @@ -46,6 +46,7 @@ namespace mlpack { * @tparam EnvironmentType The environment of the reinforcement learning task. * @tparam QNetworkType The network used to estimate the critic's Q-values. * @tparam PolicyNetworkType The network to compute action value. + * @tparam NoiseType The noise to add for exploration. * @tparam UpdaterType How to apply gradients when training. * @tparam ReplayType Experience replay method. */ @@ -53,6 +54,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType = RandomReplay > @@ -75,6 +77,7 @@ class DDPG * @param config Hyper-parameters for training. * @param learningQNetwork The network to compute action value. * @param policyNetwork The network to produce an action given a state. + * @param noise The noise instance for exploration. * @param replayMethod Experience replay method. * @param qNetworkUpdater How to apply gradients to Q network when training. * @param policyNetworkUpdater How to apply gradients to policy network @@ -84,6 +87,7 @@ class DDPG DDPG(TrainingConfig& config, QNetworkType& learningQNetwork, PolicyNetworkType& policyNetwork, + NoiseType& noise, ReplayType& replayMethod, UpdaterType qNetworkUpdater = UpdaterType(), UpdaterType policyNetworkUpdater = UpdaterType(), @@ -150,6 +154,9 @@ class DDPG //! Locally-stored policy network. PolicyNetworkType& policyNetwork; + //! Locally-stored noise instance. + NoiseType& noise; + //! Locally-stored target policy network. PolicyNetworkType targetPNetwork; diff --git a/src/mlpack/methods/reinforcement_learning/ddpg_impl.hpp b/src/mlpack/methods/reinforcement_learning/ddpg_impl.hpp index 380ee9756f..d555aab9ec 100644 --- a/src/mlpack/methods/reinforcement_learning/ddpg_impl.hpp +++ b/src/mlpack/methods/reinforcement_learning/ddpg_impl.hpp @@ -23,6 +23,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -30,11 +31,13 @@ DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::DDPG(TrainingConfig& config, QNetworkType& learningQNetwork, PolicyNetworkType& policyNetwork, + NoiseType& noise, ReplayType& replayMethod, UpdaterType qNetworkUpdater, UpdaterType policyNetworkUpdater, @@ -42,6 +45,7 @@ DDPG< config(config), learningQNetwork(learningQNetwork), policyNetwork(policyNetwork), + noise(noise), replayMethod(replayMethod), qNetworkUpdater(std::move(qNetworkUpdater)), #if ENS_VERSION_MAJOR >= 2 @@ -55,6 +59,9 @@ DDPG< totalSteps(0), deterministic(false) { + // Reset the noise instance. + noise.reset(); + // Set up q-learning and policy networks. targetPNetwork = policyNetwork; targetQNetwork = learningQNetwork; @@ -106,6 +113,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -113,6 +121,7 @@ DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::~DDPG() @@ -127,6 +136,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -134,6 +144,7 @@ void DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::SoftUpdate(double rho) @@ -148,6 +159,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -155,6 +167,7 @@ void DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::Update() @@ -255,6 +268,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -262,6 +276,7 @@ void DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::SelectAction() @@ -272,9 +287,9 @@ void DDPG< if (!deterministic) { - arma::colvec noise = arma::randn(outputAction.n_rows) * 0.1; - noise = arma::clamp(noise, -0.25, 0.25); - outputAction = outputAction + noise; + arma::colvec sample = noise.sample() * 0.1; + sample = arma::clamp(sample, -0.25, 0.25); + outputAction = outputAction + sample; } action.action = arma::conv_to>::from(outputAction); } @@ -283,6 +298,7 @@ template < typename EnvironmentType, typename QNetworkType, typename PolicyNetworkType, + typename NoiseType, typename UpdaterType, typename ReplayType > @@ -290,6 +306,7 @@ double DDPG< EnvironmentType, QNetworkType, PolicyNetworkType, + NoiseType, UpdaterType, ReplayType >::Episode() diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index bdbc527e14..650c95017c 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -598,9 +598,19 @@ TEST_CASE("PendulumWithDDPG", "[QLearningTest]") qNetwork.Add(new ReLU()); qNetwork.Add(new Linear(1)); + // Set up the OUNoise parameters. + int size = 1; + double mu = 0.0; + double theta = 1.0; + double sigma = 0.1; + + // Create an instance of the OUNoise class. + OUNoise ouNoise(size, mu, theta, sigma); + // Set up Deep Deterministic Policy Gradient agent. - DDPG - agent(config, qNetwork, policyNetwork, replayMethod); + DDPG + agent(config, qNetwork, policyNetwork, ouNoise, replayMethod); converged = testAgent(agent, -900, 500, 10); if (converged) @@ -633,10 +643,19 @@ TEST_CASE("DDPGForMultipleActions", "[QLearningTest]") config.TargetNetworkSyncInterval() = 1; config.UpdateInterval() = 3; + // Set up the OUNoise parameters. + int size = 4; + double mu = 0.0; + double theta = 1.0; + double sigma = 0.1; + + // Create an instance of the OUNoise class. + OUNoise ouNoise(size, mu, theta, sigma); + // Set up the DDPG agent. DDPG, decltype(qNetwork), decltype(policyNetwork), - AdamUpdate> - agent(config, qNetwork, policyNetwork, replayMethod); + OUNoise, AdamUpdate> + agent(config, qNetwork, policyNetwork, ouNoise, replayMethod); agent.State().Data() = arma::randu (ContinuousActionEnv<3, 4>::State::dimension, 1); From 4646ec122d0be6ea231fd3d6ea417018eadcbf09 Mon Sep 17 00:00:00 2001 From: Tarek Date: Tue, 20 Jun 2023 18:30:24 +0300 Subject: [PATCH 03/11] include Ornstein-Uhlenbeck noise in history Signed-off-by: Tarek --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 3ba0458995..23bf3050a2 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Reinforcement Learning: Ornstein-Uhlenbeck noise (#3499). + * Reinforcement Learning: Deep Deterministic Policy Gradient (#3494). ### mlpack 4.2.0 From 183396e51a6771d5d2b43f22b0d2a9a91785e533 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Thu, 22 Jun 2023 11:29:03 -0500 Subject: [PATCH 04/11] fix: append existing PYTHONPATH to CMAKE_PYTHON_PATH This addresses #3500. It approaches it slightly differently than the code included there, in order to behave correctly when `PYTHONPATH` is not set and to remain within the CMake version 3.6 minimum (prevents `JOIN`). --- src/mlpack/bindings/python/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mlpack/bindings/python/CMakeLists.txt b/src/mlpack/bindings/python/CMakeLists.txt index e4b3474474..971cf727d8 100644 --- a/src/mlpack/bindings/python/CMakeLists.txt +++ b/src/mlpack/bindings/python/CMakeLists.txt @@ -203,6 +203,9 @@ execute_process(COMMAND ${PYTHON_EXECUTABLE} "${PYTHON_INSTALL_PREFIX}" OUTPUT_VARIABLE CMAKE_PYTHON_PATH) string(STRIP "${CMAKE_PYTHON_PATH}" CMAKE_PYTHON_PATH) +if (DEFINED ENV{PYTHONPATH}) + string(APPEND CMAKE_PYTHON_PATH : $ENV{PYTHONPATH}) +endif () install(CODE "set(ENV{PYTHONPATH} ${CMAKE_PYTHON_PATH})") install(CODE "set(PYTHON_EXECUTABLE \"${PYTHON_EXECUTABLE}\")") install(CODE "set(CMAKE_BINARY_DIR \"${CMAKE_BINARY_DIR}\")") From af3b3bded4c06065d76b908c478501cc9a71ee7c Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Fri, 23 Jun 2023 08:27:12 -0500 Subject: [PATCH 05/11] fix: style python/CMakeLists.txt Co-authored-by: Ryan Curtin --- src/mlpack/bindings/python/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/bindings/python/CMakeLists.txt b/src/mlpack/bindings/python/CMakeLists.txt index 971cf727d8..33d184909c 100644 --- a/src/mlpack/bindings/python/CMakeLists.txt +++ b/src/mlpack/bindings/python/CMakeLists.txt @@ -204,7 +204,7 @@ execute_process(COMMAND ${PYTHON_EXECUTABLE} OUTPUT_VARIABLE CMAKE_PYTHON_PATH) string(STRIP "${CMAKE_PYTHON_PATH}" CMAKE_PYTHON_PATH) if (DEFINED ENV{PYTHONPATH}) - string(APPEND CMAKE_PYTHON_PATH : $ENV{PYTHONPATH}) + string(APPEND CMAKE_PYTHON_PATH : $ENV{PYTHONPATH}) endif () install(CODE "set(ENV{PYTHONPATH} ${CMAKE_PYTHON_PATH})") install(CODE "set(PYTHON_EXECUTABLE \"${PYTHON_EXECUTABLE}\")") From 9b89f79e01d3addee35b5db0804b9244b57fe101 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 4 Jul 2023 10:08:54 -0400 Subject: [PATCH 06/11] Shorten filename for CRAN tar limits. (#3507) --- ...nuous_double_pole_cart.hpp => cont_double_pole_cart.hpp} | 6 +++--- .../reinforcement_learning/environment/environment.hpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/mlpack/methods/reinforcement_learning/environment/{continuous_double_pole_cart.hpp => cont_double_pole_cart.hpp} (97%) diff --git a/src/mlpack/methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/cont_double_pole_cart.hpp similarity index 97% rename from src/mlpack/methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp rename to src/mlpack/methods/reinforcement_learning/environment/cont_double_pole_cart.hpp index c68b185e80..0cdae20c64 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/cont_double_pole_cart.hpp @@ -1,5 +1,5 @@ /** - * @file methods/reinforcement_learning/environment/continuous_double_pole_cart.hpp + * @file methods/reinforcement_learning/environment/cont_double_pole_cart.hpp * @author Rahul Ganesh Prabhu * * This file is an implementation of Continuous Double Pole Cart Balancing @@ -11,8 +11,8 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_DOUBLE_POLE_CART_HPP -#define MLPACK_METHODS_RL_ENVIRONMENT_CONTINUOUS_DOUBLE_POLE_CART_HPP +#ifndef MLPACK_METHODS_RL_ENVIRONMENT_CONT_DOUBLE_POLE_CART_HPP +#define MLPACK_METHODS_RL_ENVIRONMENT_CONT_DOUBLE_POLE_CART_HPP #include diff --git a/src/mlpack/methods/reinforcement_learning/environment/environment.hpp b/src/mlpack/methods/reinforcement_learning/environment/environment.hpp index 4df14be356..f7a3a935ed 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/environment.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/environment.hpp @@ -15,7 +15,7 @@ #include "env_type.hpp" #include "acrobot.hpp" #include "cart_pole.hpp" -#include "continuous_double_pole_cart.hpp" +#include "cont_double_pole_cart.hpp" #include "continuous_mountain_car.hpp" #include "double_pole_cart.hpp" #include "ftn.hpp" From a7c5a3ba2323df5769d08fc28b21ec65cd2d2bf5 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 7 Jul 2023 11:04:02 -0400 Subject: [PATCH 07/11] Add ClassProbabilities() member to DecisionTree. --- src/mlpack/methods/decision_tree/decision_tree.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index 695c8dc152..dd1a486d0b 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -470,6 +470,11 @@ class DecisionTree : //! trained tree). size_t SplitDimension() const { return splitDimension; } + //! Get the class probabilities, if this is a leaf node in the trained tree. + //! Note that if this is not a leaf, then this may contain arbitrary + //! information used by the split in the tree! + const arma::vec& ClassProbabilities() const { return classProbabilities; } + /** * Given a point and that this node is not a leaf, calculate the index of the * child node this point would go towards. This method is primarily used by From f91697ce1664374b59c05b4215d124366cc22430 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 7 Jul 2023 11:06:58 -0400 Subject: [PATCH 08/11] Update HISTORY.md. --- HISTORY.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 23bf3050a2..682d657661 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,9 +1,12 @@ ### mlpack ?.?.? ###### ????-??-?? * Reinforcement Learning: Ornstein-Uhlenbeck noise (#3499). - + * Reinforcement Learning: Deep Deterministic Policy Gradient (#3494). - + + * Add `ClassProbabilities()` member to `DecisionTree` so that the internal + details of trees can be more easily inspected (#3511). + ### mlpack 4.2.0 ###### 2023-06-14 * Adapt C_ReLU, ReLU6, FlexibleReLU layer for new neural network API (#3445). From f2d8f51f62d5b9ad42f4b8c8106bb0dacbc6ee81 Mon Sep 17 00:00:00 2001 From: Tarek Date: Fri, 7 Jul 2023 19:08:05 +0300 Subject: [PATCH 09/11] feat(rl): implement Twin Delayed Deep Deterministic policy gradient Signed-off-by: Tarek --- .../reinforcement_learning.hpp | 3 +- .../methods/reinforcement_learning/td3.hpp | 194 ++++++++++ .../reinforcement_learning/td3_impl.hpp | 362 ++++++++++++++++++ src/mlpack/tests/q_learning_test.cpp | 86 +++++ 4 files changed, 644 insertions(+), 1 deletion(-) create mode 100644 src/mlpack/methods/reinforcement_learning/td3.hpp create mode 100644 src/mlpack/methods/reinforcement_learning/td3_impl.hpp diff --git a/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp b/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp index 6af0276800..806852c790 100644 --- a/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp +++ b/src/mlpack/methods/reinforcement_learning/reinforcement_learning.hpp @@ -22,7 +22,8 @@ #include "training_config.hpp" #include "async_learning.hpp" #include "q_learning.hpp" -#include "sac.hpp" #include "ddpg.hpp" +#include "td3.hpp" +#include "sac.hpp" #endif diff --git a/src/mlpack/methods/reinforcement_learning/td3.hpp b/src/mlpack/methods/reinforcement_learning/td3.hpp new file mode 100644 index 0000000000..ef09f363cd --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/td3.hpp @@ -0,0 +1,194 @@ +/** + * @file methods/reinforcement_learning/td3.hpp + * @author Tarek Elsayed + * + * This file is the definition of TD3 class, which implements the + * Twin Delayed Deep Deterministic Policy Gradient algorithm. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_TD3_HPP +#define MLPACK_METHODS_RL_TD3_HPP + +#include +#include +#include + +#include "replay/replay.hpp" +#include "training_config.hpp" + +namespace mlpack { + +/** + * Implementation of Twin Delayed Deep Deterministic policy gradient, + * a model-free off-policy actor-critic based algorithm. + * + * For more details, see the following: + * @code + * @misc{Fujimoto et al, 2018, + * author = {Scott Fujimoto, + * Herke van Hoof, + * David Meger}, + * title = {Addressing Function Approximation Error in + * Actor-Critic Methods}, + * year = {2018}, + * url = {https://arxiv.org/abs/1802.09477} + * } + * @endcode + * + * @tparam EnvironmentType The environment of the reinforcement learning task. + * @tparam QNetworkType The network used to estimate the critic's Q-values. + * @tparam PolicyNetworkType The network to compute action value. + * @tparam UpdaterType How to apply gradients when training. + * @tparam ReplayType Experience replay method. + */ +template < + typename EnvironmentType, + typename QNetworkType, + typename PolicyNetworkType, + typename UpdaterType, + typename ReplayType = RandomReplay +> +class TD3 +{ + public: + //! Convenient typedef for state. + using StateType = typename EnvironmentType::State; + + //! Convenient typedef for action. + using ActionType = typename EnvironmentType::Action; + + /** + * Create the TD3 object with given settings. + * + * If you want to pass in a parameter and discard the original parameter + * object, you can directly pass the parameter, as the constructor takes + * a reference. This avoids unnecessary copy. + * + * @param config Hyper-parameters for training. + * @param learningQNetwork The network to compute action value. + * @param policyNetwork The network to produce an action given a state. + * @param replayMethod Experience replay method. + * @param qNetworkUpdater How to apply gradients to Q network when training. + * @param policyNetworkUpdater How to apply gradients to policy network + * when training. + * @param environment Reinforcement learning task. + */ + TD3(TrainingConfig& config, + QNetworkType& learningQNetwork, + PolicyNetworkType& policyNetwork, + ReplayType& replayMethod, + UpdaterType qNetworkUpdater = UpdaterType(), + UpdaterType policyNetworkUpdater = UpdaterType(), + EnvironmentType environment = EnvironmentType()); + + /** + * Clean memory. + */ + ~TD3(); + + /** + * Softly update the target networks` parameters from the learning networks` + * parameters. + * + * @param rho How "softly" should the parameters be copied. + * */ + void SoftUpdate(double rho); + + /** + * Update the Q and policy networks. + * */ + void Update(); + + /** + * Select an action, given an agent. + */ + void SelectAction(); + + /** + * Execute an episode. + * @return Return of the episode. + */ + double Episode(); + + //! Modify total steps from beginning. + size_t& TotalSteps() { return totalSteps; } + //! Get total steps from beginning. + const size_t& TotalSteps() const { return totalSteps; } + + //! Modify the state of the agent. + StateType& State() { return state; } + //! Get the state of the agent. + const StateType& State() const { return state; } + + //! Get the action of the agent. + const ActionType& Action() const { return action; } + + //! Modify the training mode / test mode indicator. + bool& Deterministic() { return deterministic; } + //! Get the indicator of training mode / test mode. + const bool& Deterministic() const { return deterministic; } + + + private: + //! Locally-stored hyper-parameters. + TrainingConfig& config; + + //! Locally-stored learning Q1 and Q2 network. + QNetworkType& learningQ1Network; + QNetworkType learningQ2Network; + + //! Locally-stored target Q1 and Q2 network. + QNetworkType targetQ1Network; + QNetworkType targetQ2Network; + + //! Locally-stored policy network. + PolicyNetworkType& policyNetwork; + + //! Locally-stored target policy network. + PolicyNetworkType targetPNetwork; + + //! Locally-stored experience method. + ReplayType& replayMethod; + + //! Locally-stored updater. + UpdaterType qNetworkUpdater; + #if ENS_VERSION_MAJOR >= 2 + typename UpdaterType::template Policy* + qNetworkUpdatePolicy; + #endif + + //! Locally-stored updater. + UpdaterType policyNetworkUpdater; + #if ENS_VERSION_MAJOR >= 2 + typename UpdaterType::template Policy* + policyNetworkUpdatePolicy; + #endif + + //! Locally-stored reinforcement learning task. + EnvironmentType environment; + + //! Total steps from the beginning of the task. + size_t totalSteps; + + //! Locally-stored current state of the agent. + StateType state; + + //! Locally-stored action of the agent. + ActionType action; + + //! Locally-stored flag indicating training mode or test mode. + bool deterministic; + + //! Locally-stored loss function. + MeanSquaredError lossFunction; +}; + +} // namespace mlpack + +// Include implementation +#include "td3_impl.hpp" +#endif diff --git a/src/mlpack/methods/reinforcement_learning/td3_impl.hpp b/src/mlpack/methods/reinforcement_learning/td3_impl.hpp new file mode 100644 index 0000000000..01e3229a6d --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/td3_impl.hpp @@ -0,0 +1,362 @@ +/** + * @file methods/reinforcement_learning/td3_impl.hpp + * @author Tarek Elsayed + * + * This file is the implementation of TD3 class, which implements the + * Twin Delayed Deep Deterministic Policy Gradient algorithm. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_td3_IMPL_HPP +#define MLPACK_METHODS_RL_td3_IMPL_HPP + +#include + +#include "td3.hpp" + +namespace mlpack { + +template < + typename EnvironmentType, + typename QNetworkType, + typename PolicyNetworkType, + typename UpdaterType, + typename ReplayType +> +TD3< + EnvironmentType, + QNetworkType, + PolicyNetworkType, + UpdaterType, + ReplayType +>::TD3(TrainingConfig& config, + QNetworkType& learningQ1Network, + PolicyNetworkType& policyNetwork, + ReplayType& replayMethod, + UpdaterType qNetworkUpdater, + UpdaterType policyNetworkUpdater, + EnvironmentType environment): + config(config), + learningQ1Network(learningQ1Network), + policyNetwork(policyNetwork), + replayMethod(replayMethod), + qNetworkUpdater(std::move(qNetworkUpdater)), + #if ENS_VERSION_MAJOR >= 2 + qNetworkUpdatePolicy(NULL), + #endif + policyNetworkUpdater(std::move(policyNetworkUpdater)), + #if ENS_VERSION_MAJOR >= 2 + policyNetworkUpdatePolicy(NULL), + #endif + environment(std::move(environment)), + totalSteps(0), + deterministic(false) +{ + // Set up q-learning and policy networks. + targetPNetwork = policyNetwork; + targetQ1Network = learningQ1Network; + learningQ2Network = learningQ1Network; + targetQ2Network = learningQ2Network; + + // Reset all the networks. + // Note: the q and policy networks have an if condition before reset. + // This is because we don't want to reset a loaded(possibly pretrained) model + // passed using this constructor. + const size_t envSampleSize = environment.InitialSample().Encode().n_elem; + if (policyNetwork.Parameters().n_elem != envSampleSize) + policyNetwork.Reset(envSampleSize); + + targetPNetwork.Reset(envSampleSize); + + const size_t networkSize = envSampleSize + + policyNetwork.Network()[policyNetwork.Network().size() - 1]->OutputSize(); + + if (learningQ1Network.Parameters().n_elem != networkSize) + { + learningQ1Network.Reset(networkSize); + learningQ2Network.Reset(networkSize); + } + targetQ1Network.Reset(networkSize); + targetQ2Network.Reset(networkSize); + + #if ENS_VERSION_MAJOR == 1 + this->qNetworkUpdater.Initialize(learningQ1Network.Parameters().n_rows, + learningQ1Network.Parameters().n_cols); + #else + this->qNetworkUpdatePolicy = new typename UpdaterType::template + Policy(this->qNetworkUpdater, + learningQ1Network.Parameters().n_rows, + learningQ1Network.Parameters().n_cols); + #endif + + #if ENS_VERSION_MAJOR == 1 + this->policyNetworkUpdater.Initialize(policyNetwork.Parameters().n_rows, + policyNetwork.Parameters().n_cols); + #else + this->policyNetworkUpdatePolicy = new typename UpdaterType::template + Policy(this->policyNetworkUpdater, + policyNetwork.Parameters().n_rows, + policyNetwork.Parameters().n_cols); + #endif + + // Copy over the learning networks to their respective target networks. + targetQ1Network.Parameters() = learningQ1Network.Parameters(); + targetQ2Network.Parameters() = learningQ2Network.Parameters(); + targetPNetwork.Parameters() = policyNetwork.Parameters(); +} + +template < + typename EnvironmentType, + typename QNetworkType, + typename PolicyNetworkType, + typename UpdaterType, + typename ReplayType +> +TD3< + EnvironmentType, + QNetworkType, + PolicyNetworkType, + UpdaterType, + ReplayType +>::~TD3() +{ + #if ENS_VERSION_MAJOR >= 2 + delete qNetworkUpdatePolicy; + delete policyNetworkUpdatePolicy; + #endif +} + +template < + typename EnvironmentType, + typename QNetworkType, + typename PolicyNetworkType, + typename UpdaterType, + typename ReplayType +> +void TD3< + EnvironmentType, + QNetworkType, + PolicyNetworkType, + UpdaterType, + ReplayType +>::SoftUpdate(double rho) +{ + targetQ1Network.Parameters() = (1 - rho) * targetQ1Network.Parameters() + + rho * learningQ1Network.Parameters(); + targetQ2Network.Parameters() = (1 - rho) * targetQ2Network.Parameters() + + rho * learningQ2Network.Parameters(); + targetPNetwork.Parameters() = (1 - rho) * targetPNetwork.Parameters() + + rho * policyNetwork.Parameters(); +} + +template < + typename EnvironmentType, + typename QNetworkType, + typename PolicyNetworkType, + typename UpdaterType, + typename ReplayType +> +void TD3< + EnvironmentType, + QNetworkType, + PolicyNetworkType, + UpdaterType, + ReplayType +>::Update() +{ + // Sample from previous experience. + arma::mat sampledStates; + std::vector sampledActions; + arma::rowvec sampledRewards; + arma::mat sampledNextStates; + arma::irowvec isTerminal; + + replayMethod.Sample(sampledStates, sampledActions, sampledRewards, + sampledNextStates, isTerminal); + + // Critic network update. + + // Use the target actor to obtain the next actions. + arma::mat nextStateActions; + targetPNetwork.Predict(sampledNextStates, nextStateActions); + + // Compute the estimated next Q-values using the target Q-networks. + arma::mat targetQInput = arma::join_vert(nextStateActions, + sampledNextStates); + arma::rowvec Q1, Q2; + targetQ1Network.Predict(targetQInput, Q1); + targetQ2Network.Predict(targetQInput, Q2); + arma::rowvec nextQ = sampledRewards + config.Discount() * ((1 - isTerminal) + % arma::min(Q1, Q2)); + + arma::mat sampledActionValues(action.size, sampledActions.size()); + for (size_t i = 0; i < sampledActions.size(); i++) + sampledActionValues.col(i) = arma::conv_to::from + (sampledActions[i].action); + arma::mat learningQInput = arma::join_vert(sampledActionValues, + sampledStates); + learningQ1Network.Forward(learningQInput, Q1); + learningQ2Network.Forward(learningQInput, Q2); + + arma::mat gradQ1Loss, gradQ2Loss; + lossFunction.Backward(Q1, nextQ, gradQ1Loss); + lossFunction.Backward(Q2, nextQ, gradQ2Loss); + + // Sum both losses + arma::mat combinedLoss = gradQ1Loss + gradQ2Loss; + + // Update the critic networks. + arma::mat gradientQ1, gradientQ2; + learningQ1Network.Backward(learningQInput, combinedLoss, gradientQ1); + learningQ2Network.Backward(learningQInput, combinedLoss, gradientQ2); + #if ENS_VERSION_MAJOR == 1 + qNetworkUpdater.Update(learningQ1Network.Parameters(), config.StepSize(), + gradientQ1); + #else + qNetworkUpdatePolicy->Update(learningQ1Network.Parameters(), + config.StepSize(), gradientQ1); + #endif + #if ENS_VERSION_MAJOR == 1 + qNetworkUpdater.Update(learningQ2Network.Parameters(), config.StepSize(), + gradientQ2); + #else + qNetworkUpdatePolicy->Update(learningQ2Network.Parameters(), + config.StepSize(), gradientQ2); + #endif + + // Actor network update. + + if (totalSteps % config.TargetNetworkSyncInterval() == 0) + { + // Get the size of the first hidden layer in the Q network. + size_t hidden1 = learningQ1Network.Network()[0]->OutputSize(); + + arma::mat gradient; + for (size_t i = 0; i < sampledStates.n_cols; i++) + { + arma::mat grad, gradQ, q; + arma::colvec singleState = sampledStates.col(i); + arma::colvec singlePi; + policyNetwork.Forward(singleState, singlePi); + arma::colvec input = arma::join_vert(singlePi, singleState); + arma::mat weightLastLayer; + + // Note that we can use an empty matrix for the backwards pass, since the + // networks use EmptyLoss. + learningQ1Network.Forward(input, q); + learningQ1Network.Backward(input, arma::mat("-1"), gradQ); + weightLastLayer = arma::reshape(learningQ1Network.Parameters(). + rows(0, hidden1 * singlePi.n_rows - 1), hidden1, singlePi.n_rows); + + arma::colvec gradQBias = gradQ(input.n_rows * hidden1, 0, + arma::size(hidden1, 1)); + arma::mat gradPolicy = weightLastLayer.t() * gradQBias; + policyNetwork.Backward(singleState, gradPolicy, grad); + if (i == 0) + { + gradient.copy_size(grad); + gradient.fill(0.0); + } + gradient += grad; + } + gradient /= sampledStates.n_cols; + + #if ENS_VERSION_MAJOR == 1 + policyUpdater.Update(policyNetwork.Parameters(), config.StepSize(), gradient); + #else + policyNetworkUpdatePolicy->Update(policyNetwork.Parameters(), + config.StepSize(), gradient); + #endif + + // Update target networks + SoftUpdate(config.Rho()); + } +} + +template < + typename EnvironmentType, + typename QNetworkType, + typename PolicyNetworkType, + typename UpdaterType, + typename ReplayType +> +void TD3< + EnvironmentType, + QNetworkType, + PolicyNetworkType, + UpdaterType, + ReplayType +>::SelectAction() +{ + // Get the action at current state, from policy. + arma::colvec outputAction; + policyNetwork.Predict(state.Encode(), outputAction); + + if (!deterministic) + { + arma::colvec noise = arma::randn(outputAction.n_rows) * 0.1; + noise = arma::clamp(noise, -0.25, 0.25); + outputAction = outputAction + noise; + } + action.action = arma::conv_to>::from(outputAction); +} + +template < + typename EnvironmentType, + typename QNetworkType, + typename PolicyNetworkType, + typename UpdaterType, + typename ReplayType +> +double TD3< + EnvironmentType, + QNetworkType, + PolicyNetworkType, + UpdaterType, + ReplayType +>::Episode() +{ + // Get the initial state from environment. + state = environment.InitialSample(); + + // Track the steps in this episode. + size_t steps = 0; + + // Track the return of this episode. + double totalReturn = 0.0; + + // Running until get to the terminal state. + while (!environment.IsTerminal(state)) + { + if (config.StepLimit() && steps >= config.StepLimit()) + break; + SelectAction(); + + // Interact with the environment to advance to next state. + StateType nextState; + double reward = environment.Sample(state, action, nextState); + + totalReturn += reward; + steps++; + totalSteps++; + + // Store the transition for replay. + replayMethod.Store(state, action, reward, nextState, + environment.IsTerminal(nextState), config.Discount()); + + // Update current state. + state = nextState; + + if (deterministic || totalSteps < config.ExplorationSteps()) + continue; + for (size_t i = 0; i < config.UpdateInterval(); i++) + Update(); + } + return totalReturn; +} + +} // namespace mlpack +#endif diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 650c95017c..10b7c44723 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -693,3 +693,89 @@ TEST_CASE("OUNoiseTest", "[QLearningTest]") bool isNotEqual = arma::any(sample != state); REQUIRE(isNotEqual); } + +//! Test TD3 on Pendulum task. +TEST_CASE("PendulumWithTD3", "[QLearningTest]") +{ + // It isn't guaranteed that the network will converge in the specified number + // of iterations using random weights. + bool converged = false; + for (size_t trial = 0; trial < 8; ++trial) + { + Log::Debug << "Trial number: " << trial << std::endl; + // Set up the replay method. + RandomReplay replayMethod(32, 10000); + + TrainingConfig config; + config.StepSize() = 0.001; + config.TargetNetworkSyncInterval() = 2; + config.UpdateInterval() = 3; + + // Set up Actor network. + FFN + policyNetwork(EmptyLoss(), GaussianInitialization(0, 0.1)); + policyNetwork.Add(new Linear(128)); + policyNetwork.Add(new ReLU()); + policyNetwork.Add(new Linear(1)); + policyNetwork.Add(new TanH()); + + // Set up Critic network. + FFN + qNetwork(EmptyLoss(), GaussianInitialization(0, 0.1)); + qNetwork.Add(new Linear(128)); + qNetwork.Add(new ReLU()); + qNetwork.Add(new Linear(1)); + + // Set up Twin Delayed Deep Deterministic policy gradient agent. + TD3 + agent(config, qNetwork, policyNetwork, replayMethod); + + converged = testAgent(agent, -900, 500, 10); + if (converged) + break; + } + REQUIRE(converged); +} + +//! A test to ensure TD3 works with multiple actions in action space. +TEST_CASE("TD3ForMultipleActions", "[QLearningTest]") +{ + FFN + policyNetwork(EmptyLoss(), GaussianInitialization(0, 0.1)); + policyNetwork.Add(new Linear(128)); + policyNetwork.Add(new ReLU()); + policyNetwork.Add(new Linear(4)); + policyNetwork.Add(new TanH()); + + FFN + qNetwork(EmptyLoss(), GaussianInitialization(0, 0.1)); + qNetwork.Add(new Linear(128)); + qNetwork.Add(new ReLU()); + qNetwork.Add(new Linear(1)); + + // Set up the replay method. + RandomReplay> replayMethod(32, 10000); + + TrainingConfig config; + config.StepSize() = 0.001; + config.TargetNetworkSyncInterval() = 2; + config.UpdateInterval() = 3; + + // Set up the TD3 agent. + TD3, decltype(qNetwork), decltype(policyNetwork), + AdamUpdate> + agent(config, qNetwork, policyNetwork, replayMethod); + + agent.State().Data() = arma::randu + (ContinuousActionEnv<3, 4>::State::dimension, 1); + agent.SelectAction(); + + // Test to check if the action dimension given by the agent is correct. + REQUIRE(agent.Action().action.size() == 4); + + replayMethod.Store(agent.State(), agent.Action(), 1, agent.State(), 1, 0.99); + agent.TotalSteps()++; + agent.Update(); + // If the agent is able to reach this point of the test, it is assured + // that the agent can handle multiple actions in continuous space. +} From 9feb73e1674960ee7337de56cd30a7f5474df824 Mon Sep 17 00:00:00 2001 From: Tarek Date: Fri, 7 Jul 2023 19:13:44 +0300 Subject: [PATCH 10/11] feat(history): add pr 3512 to history.md Signed-off-by: Tarek --- HISTORY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 23bf3050a2..1c12b20e5e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,8 @@ ### mlpack ?.?.? ###### ????-??-?? + * Reinforcement Learning: Twin Delayed Deep Deterministic + Policy Gradient (#3512). + * Reinforcement Learning: Ornstein-Uhlenbeck noise (#3499). * Reinforcement Learning: Deep Deterministic Policy Gradient (#3494). From 5e8beeb0be42ffa12c10659088ce02f923a04549 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 14 Jul 2023 02:16:29 -0400 Subject: [PATCH 11/11] Python binding installation fixes (#3505) --- .ci/linux-steps.yaml | 2 +- README.md | 1 + src/mlpack/bindings/python/CMakeLists.txt | 8 ++++++-- src/mlpack/bindings/python/PythonInstall.cmake | 9 ++++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index 6ad21cb7d6..ebaa342552 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -27,7 +27,7 @@ steps: if [ "$(binding)" == "python" ]; then export PYBIN=$(which python) $PYBIN -m pip install --upgrade pip - $PYBIN -m pip install --upgrade --ignore-installed setuptools cython pandas + $PYBIN -m pip install --upgrade --ignore-installed setuptools cython pandas wheel fi if [ "a$(julia.version)" != "a" ]; then diff --git a/README.md b/README.md index 2a0dffe2e5..032c9d5a50 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,7 @@ With that in mind, if you would still like to manually build the mlpack Python bindings, first make sure that the following Python packages are installed: - setuptools + - wheel - cython >= 0.24 - numpy - pandas >= 0.15.0 diff --git a/src/mlpack/bindings/python/CMakeLists.txt b/src/mlpack/bindings/python/CMakeLists.txt index 33d184909c..b4e366db12 100644 --- a/src/mlpack/bindings/python/CMakeLists.txt +++ b/src/mlpack/bindings/python/CMakeLists.txt @@ -70,18 +70,22 @@ find_python_module(pandas 0.15.0) if (NOT PY_PANDAS) set(PY_NOT_FOUND_MSG "${PY_NOT_FOUND_MSG}\n - pandas") endif () +find_python_module(wheel) +if (NOT PY_WHEEL) + set(PY_NOT_FOUND_MSG "${PY_NOT_FOUND_MSG}\n - wheel") +endif () ## We need to check here if Python and other dependencies is even available, as ## it is require to build python-bindings. if (FORCE_BUILD_PYTHON_BINDINGS) if (NOT PYTHON_EXECUTABLE OR NOT PY_DISTUTILS OR NOT PY_CYTHON OR NOT PY_NUMPY - OR NOT PY_PANDAS) + OR NOT PY_PANDAS OR NOT PY_WHEEL) unset(BUILD_PYTHON_BINDINGS CACHE) message(FATAL_ERROR "\nCould not Build Python Bindings; the following modules are not available: ${PY_NOT_FOUND_MSG}") endif() else() if (NOT PYTHON_EXECUTABLE OR NOT PY_DISTUTILS OR NOT PY_CYTHON OR NOT PY_NUMPY - OR NOT PY_PANDAS) + OR NOT PY_PANDAS OR NOT PY_WHEEL) unset(BUILD_PYTHON_BINDINGS CACHE) not_found_return("Not building Python bindings; the following modules are not available: ${PY_NOT_FOUND_MSG}") endif() diff --git a/src/mlpack/bindings/python/PythonInstall.cmake b/src/mlpack/bindings/python/PythonInstall.cmake index 880f57cc21..49de9f03f4 100644 --- a/src/mlpack/bindings/python/PythonInstall.cmake +++ b/src/mlpack/bindings/python/PythonInstall.cmake @@ -5,16 +5,19 @@ if (DEFINED ENV{DESTDIR}) execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install --prefix=${PYTHON_INSTALL_PREFIX} - --root=$ENV{DESTDIR} . + --root=$ENV{DESTDIR} + --no-index --no-deps --no-build-isolation -f ./ . WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/" RESULT_VARIABLE setup_res) elseif (WIN32) - execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install . + execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install + --no-index --no-deps --no-build-isolation -f ./ . WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/" RESULT_VARIABLE setup_res) else () execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install - --prefix=${PYTHON_INSTALL_PREFIX} . + --prefix=${PYTHON_INSTALL_PREFIX} + --no-index --no-deps --no-build-isolation -f ./ . WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/" RESULT_VARIABLE setup_res) endif ()