From e23cc7c90139d52c5505063abe9c373f745d7800 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 25 Mar 2019 18:26:22 -0400 Subject: [PATCH 1/4] Fix pandas memory issue with copies. --- .../bindings/python/mlpack/matrix_utils.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/mlpack/bindings/python/mlpack/matrix_utils.py b/src/mlpack/bindings/python/mlpack/matrix_utils.py index 39bd235154..9a2a15868c 100644 --- a/src/mlpack/bindings/python/mlpack/matrix_utils.py +++ b/src/mlpack/bindings/python/mlpack/matrix_utils.py @@ -54,19 +54,26 @@ def to_matrix(x, dtype=np.double, copy=False): return x, False elif (isinstance(x, np.ndarray) and x.dtype == dtype and x.flags.f_contiguous): if copy: # Copy the matrix if required. - return np.ndarray(x.shape, buffer=x.flatten(), dtype=dtype, order='C').copy("C"), True + return np.ndarray(x.shape, buffer=x.flatten(), dtype=dtype, + order='C').copy("C"), True else: - return np.ndarray(x.shape, buffer=x.flatten(), dtype=dtype, order='C'), False + return np.ndarray(x.shape, buffer=x.flatten(), dtype=dtype, order='C'), \ + False else: if isinstance(x, pd.core.series.Series) or isinstance(x, pd.DataFrame): + # We can only avoid a copy if the dtype is the same and the copy flag is + # false. I'm actually not sure if this is possible, since in everything I + # have found, Pandas stores with F_CONTIGUOUS not C_CONTIGUOUS. y = x.values - if copy: # Copy the matrix if required. - return np.ndarray(y.shape, buffer=y.flatten(), dtype=dtype, order='C').copy("C"), True + if copy == False and y.dtype == dtype and y.flags.c_contiguous: + return np.ndarray(y.shape, buffer=y.flatten(), dtype=dtype, order='C'),\ + False else: - return np.ndarray(y.shape, buffer=y.flatten(), dtype=dtype, order='C'), False + # We have to make a copy or change the dtype, so just do this directly. + return np.array(y, dtype=dtype, order='C', copy=True), True else: return np.array(x, copy=True, dtype=dtype, order='C'), True - + def to_matrix_with_info(x, dtype, copy=False): """ From 8234cdbd1b346680ebcb928347a23e729f284eb0 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 25 Mar 2019 18:26:43 -0400 Subject: [PATCH 2/4] Get a new enough version of Cython (or try to). --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ff277e33dc..395f1d6fa6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ matrix: before_install: - sudo apt-get update - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost-all-dev python-pip cython python-numpy python-pandas - - sudo pip install --upgrade --ignore-installed setuptools + - sudo pip install --upgrade --ignore-installed setuptools cython - curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-6.500.5.tar.gz | tar xvz && cd armadillo* - cmake . && make && sudo make install && cd .. - sudo cp .travis/config.hpp /usr/include/armadillo_bits/config.hpp @@ -20,7 +20,7 @@ matrix: before_install: - sudo apt-get update - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost-all-dev python3-pip cython3 python3-numpy python3-pandas - - sudo pip3 install --upgrade --ignore-installed setuptools + - sudo pip3 install --upgrade --ignore-installed setuptools cython - curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-6.500.5.tar.gz | tar xvz && cd armadillo* - cmake . && make && sudo make install && cd .. - sudo cp .travis/config.hpp /usr/include/armadillo_bits/config.hpp From 99288aa08496d05efb03b6604ad0c7d3b1039557 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 25 Mar 2019 18:53:31 -0400 Subject: [PATCH 3/4] Use correct version of Python. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 395f1d6fa6..63373caaf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ matrix: include: - os: linux dist: xenial - env: CMAKE_OPTIONS="-DDEBUG=OFF -DPROFILE=OFF -DPYTHON=/usr/bin/python" + env: CMAKE_OPTIONS="-DDEBUG=OFF -DPROFILE=OFF -DPYTHON_EXECUTABLE=/usr/bin/python" before_install: - sudo apt-get update - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost-all-dev python-pip cython python-numpy python-pandas @@ -16,7 +16,7 @@ matrix: - os: linux dist: xenial - env: CMAKE_OPTIONS="-DDEBUG=OFF -DPROFILE=OFF -DPYTHON=/usr/bin/python3" + env: CMAKE_OPTIONS="-DDEBUG=OFF -DPROFILE=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3" before_install: - sudo apt-get update - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost-all-dev python3-pip cython3 python3-numpy python3-pandas From 777b83ce3b21024ef8c67d718e9ef1f2feed8593 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 26 Mar 2019 09:45:46 -0400 Subject: [PATCH 4/4] Not sure why the system pandas doesn't work. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 63373caaf9..f8291e7b41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,8 +19,8 @@ matrix: env: CMAKE_OPTIONS="-DDEBUG=OFF -DPROFILE=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3" before_install: - sudo apt-get update - - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost-all-dev python3-pip cython3 python3-numpy python3-pandas - - sudo pip3 install --upgrade --ignore-installed setuptools cython + - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost-all-dev python3-pip cython3 python3-numpy + - sudo pip3 install --upgrade --ignore-installed setuptools cython pandas - curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-6.500.5.tar.gz | tar xvz && cd armadillo* - cmake . && make && sudo make install && cd .. - sudo cp .travis/config.hpp /usr/include/armadillo_bits/config.hpp