Merge pull request #1864 from rcurtin/travis-python-fixes

Python fixes for Travis build
This commit is contained in:
Ryan Curtin
2019-04-18 10:04:55 -04:00
committed by GitHub
3 changed files with 89 additions and 119 deletions
@@ -53,12 +53,8 @@ def to_matrix(x, dtype=np.double, copy=False):
else:
return x, False
elif (isinstance(x, np.ndarray) and x.dtype == dtype and x.flags.f_contiguous):
if copy: # Copy the matrix if required.
return np.ndarray(x.shape, buffer=x.flatten(), dtype=dtype,
order='C').copy("C"), True
else:
return np.ndarray(x.shape, buffer=x.flatten(), dtype=dtype, order='C'), \
False
# A copy is always necessary here.
return x.copy("C"), True
else:
if isinstance(x, pd.core.series.Series) or isinstance(x, pd.DataFrame):
# We can only avoid a copy if the dtype is the same and the copy flag is
@@ -66,7 +62,7 @@ def to_matrix(x, dtype=np.double, copy=False):
# have found, Pandas stores with F_CONTIGUOUS not C_CONTIGUOUS.
y = x.values
if copy == False and y.dtype == dtype and y.flags.c_contiguous:
return np.ndarray(y.shape, buffer=y.flatten(), dtype=dtype, order='C'),\
return np.ndarray(y.shape, buffer=x.values, dtype=dtype, order='C'),\
False
else:
# We have to make a copy or change the dtype, so just do this directly.
@@ -62,8 +62,6 @@ void PrintInputProcessing(
* else:
* raise TypeError("'param_name' must have type 'list'!")
*/
std::cout << prefix << "# Detect if the parameter was passed; set if so."
<< std::endl;
if (!d.required)
@@ -87,8 +85,6 @@ void PrintInputProcessing(
<< "](<const string> '" << d.name << "', ";
if (GetCythonType<T>(d) == "string")
std::cout << name << ".encode(\"UTF-8\")";
else if (GetCythonType<T>(d) == "vector[string]")
std::cout << "[i.encode(\"UTF-8\") for i in " << name << "]";
else
std::cout << name;
std::cout << ")" << std::endl;
@@ -190,7 +186,6 @@ void PrintInputProcessing(
* raise TypeError("'param_name' must have type 'list'!")
*
*/
std::cout << prefix << "# Detect if the parameter was passed; set if so."
<< std::endl;
if (!d.required)
@@ -204,7 +199,12 @@ void PrintInputProcessing(
std::cout << prefix << " if isinstance(" << d.name << "[0], "
<< GetPrintableType<typename T::value_type>(d) << "):" << std::endl;
std::cout << prefix << " SetParam[" << GetCythonType<T>(d)
<< "](<const string> '" << d.name << "', " << d.name;
<< "](<const string> '" << d.name << "', ";
// Strings need special handling.
if (GetCythonType<T>(d) == "vector[string]")
std::cout << "[i.encode(\"UTF-8\") for i in " << d.name << "]";
else
std::cout << d.name;
std::cout << ")" << std::endl;
std::cout << prefix << " CLI.SetPassed(<const string> '" << d.name
<< "')" << std::endl;
@@ -225,7 +225,12 @@ void PrintInputProcessing(
std::cout << prefix << " if isinstance(" << d.name << "[0], "
<< GetPrintableType<typename T::value_type>(d) << "):" << std::endl;
std::cout << prefix << " SetParam[" << GetCythonType<T>(d)
<< "](<const string> '" << d.name << "', " << d.name;
<< "](<const string> '" << d.name << "', ";
// Strings need special handling.
if (GetCythonType<T>(d) == "vector[string]")
std::cout << "[i.encode(\"UTF-8\") for i in " << d.name << "]";
else
std::cout << d.name;
std::cout << ")" << std::endl;
std::cout << prefix << " CLI.SetPassed(<const string> '" << d.name
<< "')" << std::endl;
@@ -259,12 +264,9 @@ void PrintInputProcessing(
* param_name_tuple = to_matrix(param_name)
* if param_name_tuple[0].shape[0] == 1 or
* param_name_tuple[0].shape[1] == 1:
* param_name_reshape = param_name_tuple[0].ravel()
* param_name_mat = arma_numpy.numpy_to_mat_s(param_name_reshape,
* param_name_tuple[1])
* else:
* param_name_mat = arma_numpy.numpy_to_mat_s(param_name_tuple[0],
* param_name_tuple[1])
* param_name_tuple[0].shape = (param_name_tuple[0].size,)
* param_name_mat = arma_numpy.numpy_to_mat_s(param_name_tuple[0],
* param_name_tuple[1])
* SetParam[mat](<const string> 'param_name', dereference(param_name_mat))
* CLI.SetPassed(<const string> 'param_name')
*
@@ -279,18 +281,14 @@ void PrintInputProcessing(
std::cout << prefix << " " << d.name << "_tuple = to_matrix("
<< d.name << ", dtype=" << GetNumpyType<typename T::elem_type>()
<< ", copy=CLI.HasParam('copy_all_inputs'))" << std::endl;
std::cout << prefix << " " << "if len(" << d.name << "_tuple[0].shape"
<< ") > 1:" << std::endl;
std::cout << prefix << " " << prefix << "if " << d.name << "_tuple[0]"
std::cout << prefix << " if len(" << d.name << "_tuple[0].shape) > 1:"
<< std::endl;
std::cout << prefix << " if " << d.name << "_tuple[0]"
<< ".shape[0] == 1 or " << d.name << "_tuple[0].shape[1] == 1:"
<< std::endl;
std::cout << prefix << " " << prefix << " " << d.name
<< "_reshape = " << d.name << "_tuple[0].ravel()" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
<< "_reshape, " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " " << "else:" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
std::cout << prefix << " " << d.name << "_tuple[0].shape = ("
<< d.name << "_tuple[0].size,)" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
<< "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " SetParam[" << GetCythonType<T>(d)
@@ -306,16 +304,11 @@ void PrintInputProcessing(
std::cout << prefix << " " << d.name << "_tuple = to_matrix("
<< d.name << ", dtype=" << GetNumpyType<typename T::elem_type>()
<< ", copy=CLI.HasParam('copy_all_inputs'))" << std::endl;
std::cout << prefix << " " << "if len(" << d.name << "_tuple[0].shape"
std::cout << prefix << " if len(" << d.name << "_tuple[0].shape"
<< ") < 2:" << std::endl;
std::cout << prefix << " " << prefix << d.name
<< "_reshape = np.reshape(("<< d.name << "_tuple[0]), (" << d.name
<< "_tuple[0].shape[0] , 1))" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
<< "_reshape, " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " " << "else:" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
std::cout << prefix << " " << d.name << "_tuple[0].shape = (" << d.name
<< "_tuple[0].shape[0], 1)" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
<< "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " SetParam[" << GetCythonType<T>(d)
@@ -333,18 +326,14 @@ void PrintInputProcessing(
std::cout << prefix << " " << d.name << "_tuple = to_matrix("
<< d.name << ", dtype=" << GetNumpyType<typename T::elem_type>()
<< ", copy=CLI.HasParam('copy_all_inputs'))" << std::endl;
std::cout << prefix << " " << "if len(" << d.name << "_tuple[0].shape"
<< ") > 1:" << std::endl;
std::cout << prefix << " " << prefix << "if " << d.name << "_tuple[0]"
std::cout << prefix << " if len(" << d.name << "_tuple[0].shape) > 1:"
<< std::endl;
std::cout << prefix << " if " << d.name << "_tuple[0]"
<< ".shape[0] == 1 or " << d.name << "_tuple[0].shape[1] == 1:"
<< std::endl;
std::cout << prefix << " " << prefix << " " << d.name
<< "_reshape = " << d.name << "_tuple[0].ravel()" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
<< "_reshape, " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " " << "else:" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
std::cout << prefix << " " << d.name << "_tuple[0].shape = ("
<< d.name << "_tuple[0].size,)" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
<< "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " SetParam[" << GetCythonType<T>(d)
@@ -359,16 +348,11 @@ void PrintInputProcessing(
std::cout << prefix << " " << d.name << "_tuple = to_matrix("
<< d.name << ", dtype=" << GetNumpyType<typename T::elem_type>()
<< ", copy=CLI.HasParam('copy_all_inputs'))" << std::endl;
std::cout << prefix << " " << "if len(" << d.name << "_tuple[0].shape"
<< ") < 2:" << std::endl;
std::cout << prefix << " " << prefix << d.name
<< "_reshape = np.reshape(("<< d.name << "_tuple[0]), (" << d.name
<< "_tuple[0].shape[0] , 1))" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
<< "_reshape, " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " " << "else:" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
std::cout << prefix << " if len(" << d.name << "_tuple[0].shape) > 2:"
<< std::endl;
std::cout << prefix << " " << d.name << "_tuple[0].shape = (" << d.name
<< "_tuple[0].shape[0], 1)" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< GetArmaType<T>() << "_" << GetNumpyTypeChar<T>() << "(" << d.name
<< "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " SetParam[" << GetCythonType<T>(d)
@@ -381,6 +365,7 @@ void PrintInputProcessing(
}
std::cout << std::endl;
}
/**
* Print input processing for a serializable type.
*/
@@ -471,11 +456,13 @@ void PrintInputProcessing(
/** We want to generate code like the following:
*
* if param_name is not None:
* param_name_tuple = to_matrix_with_info(param_name)
* param_name_mat = arma_numpy.numpy_to_matrix_d(param_name_tuple[0])
* SetParamWithInfo[mat](<const string> 'param_name',
* dereference(param_name_mat), &param_name_tuple[1][0])
* CLI.SetPassed(<const string> 'param_name')
* param_name_tuple = to_matrix_with_info(param_name)
* if len(param_name_tuple[0].shape) < 2:
* param_name_tuple[0].shape = (param_name_tuple[0].size,)
* param_name_mat = arma_numpy.numpy_to_matrix_d(param_name_tuple[0])
* SetParamWithInfo[mat](<const string> 'param_name',
* dereference(param_name_mat), &param_name_tuple[1][0])
* CLI.SetPassed(<const string> 'param_name')
*/
std::cout << prefix << "cdef np.ndarray " << d.name << "_dims" << std::endl;
std::cout << prefix << "# Detect if the parameter was passed; set if so."
@@ -486,19 +473,12 @@ void PrintInputProcessing(
std::cout << prefix << " " << d.name << "_tuple = to_matrix_with_info("
<< d.name << ", dtype=np.double, copy=CLI.HasParam('copy_all_inputs'))"
<< std::endl;
std::cout << prefix << " " << "if len(" << d.name << "_tuple[0].shape"
std::cout << prefix << " if len(" << d.name << "_tuple[0].shape"
<< ") < 2:" << std::endl;
std::cout << prefix << " " << prefix << d.name
<< "_reshape = np.reshape(("<< d.name << "_tuple[0]), (" << d.name
<< "_tuple[0].shape[0] , 1))" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< "mat_d" << "(" << d.name
<< "_reshape, " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " " << "else:" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< "mat_d" << "(" << d.name
<< "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " " << d.name << "_tuple[0].shape = (" << d.name
<< "_tuple[0].shape[0], 1)" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_mat_d("
<< d.name << "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " " << d.name << "_dims = " << d.name
<< "_tuple[2]" << std::endl;
std::cout << prefix << " SetParamWithInfo[arma.Mat[double]](<const "
@@ -510,29 +490,23 @@ void PrintInputProcessing(
}
else
{
std::cout << prefix << " " << d.name << "_tuple = to_matrix_with_info("
<< d.name << ", dtype=np.double, copy=CLI.HasParam('copy_all_inputs'))"
std::cout << prefix << d.name << "_tuple = to_matrix_with_info(" << d.name
<< ", dtype=np.double, copy=CLI.HasParam('copy_all_inputs'))"
<< std::endl;
std::cout << prefix << " " << "if len(" << d.name << "_tuple[0].shape"
std::cout << prefix << "if len(" << d.name << "_tuple[0].shape"
<< ") < 2:" << std::endl;
std::cout << prefix << " " << prefix << d.name
<< "_reshape = np.reshape(("<< d.name << "_tuple[0]), (" << d.name
<< "_tuple[0].shape[0] , 1))" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< "mat_d" << "(" << d.name
<< "_reshape, " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " " << "else:" << std::endl;
std::cout << prefix << " " << d.name << "_mat = arma_numpy.numpy_to_"
<< "mat_d" << "(" << d.name
<< "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << " " << d.name << "_dims = " << d.name
<< "_tuple[2]" << std::endl;
std::cout << prefix << " SetParamWithInfo[arma.Mat[double]](<const "
std::cout << prefix << " " << d.name << "_tuple[0].shape = (" << d.name
<< "_tuple[0].shape[0], 1)" << std::endl;
std::cout << prefix << d.name << "_mat = arma_numpy.numpy_to_mat_d("
<< d.name << "_tuple[0], " << d.name << "_tuple[1])" << std::endl;
std::cout << prefix << d.name << "_dims = " << d.name << "_tuple[2]"
<< std::endl;
std::cout << prefix << "SetParamWithInfo[arma.Mat[double]](<const "
<< "string> '" << d.name << "', dereference(" << d.name << "_mat), "
<< "<const cbool*> " << d.name << "_dims.data)" << std::endl;
std::cout << prefix << " CLI.SetPassed(<const string> '" << d.name
<< "')" << std::endl;
std::cout << prefix << " del " << d.name << "_mat" << std::endl;
std::cout << prefix << "CLI.SetPassed(<const string> '" << d.name << "')"
<< std::endl;
std::cout << prefix << "del " << d.name << "_mat" << std::endl;
}
std::cout << std::endl;
}
@@ -100,7 +100,7 @@ class TestPythonBinding(unittest.TestCase):
and the fifth forgotten.
"""
x = np.random.rand(100, 5);
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -147,7 +147,7 @@ class TestPythonBinding(unittest.TestCase):
dimension doubled and the fifth forgotten.
"""
x = np.array(np.random.rand(100, 5), order='F');
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -192,7 +192,7 @@ class TestPythonBinding(unittest.TestCase):
Test that we can pass pandas.Series as input parameter.
"""
x = pd.Series(np.random.rand(100))
z = copy.copy(x)
z = x.copy(deep=True)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -203,7 +203,7 @@ class TestPythonBinding(unittest.TestCase):
self.assertEqual(output['smatrix_out'].dtype, np.double)
for i in range(100):
self.assertEqual(output['smatrix_out'][i,0], z.iloc[i] * 2)
self.assertEqual(output['smatrix_out'][i,0], x.iloc[i] * 2)
def testPandasSeriesMatrixForceCopy(self):
@@ -229,7 +229,7 @@ class TestPythonBinding(unittest.TestCase):
Test that we can pass pandas.Series as input parameter.
"""
x = pd.Series(np.random.randint(0, high=500, size=100))
z = copy.copy(x)
z = x.copy(deep=True)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -240,7 +240,7 @@ class TestPythonBinding(unittest.TestCase):
self.assertEqual(output['s_umatrix_out'].dtype, np.long)
for i in range(100):
self.assertEqual(output['s_umatrix_out'][i, 0], z.iloc[i] * 2)
self.assertEqual(output['s_umatrix_out'][i, 0], x.iloc[i] * 2)
def testPandasSeriesUMatrixForceCopy(self):
@@ -266,7 +266,7 @@ class TestPythonBinding(unittest.TestCase):
Test a Pandas Series input paramter
"""
x = pd.Series(np.random.rand(100))
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -303,7 +303,7 @@ class TestPythonBinding(unittest.TestCase):
and the fifth forgotten.
"""
x = pd.DataFrame(np.random.rand(100, 5))
z = copy.copy(x)
z = x.copy(deep=True)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -409,7 +409,7 @@ class TestPythonBinding(unittest.TestCase):
Same as testNumpyMatrix() but with an unsigned matrix.
"""
x = np.random.randint(0, high=500, size=[100, 5])
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -514,7 +514,7 @@ class TestPythonBinding(unittest.TestCase):
Test a column vector input parameter.
"""
x = np.random.rand(100)
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -550,7 +550,7 @@ class TestPythonBinding(unittest.TestCase):
Test an unsigned column vector input parameter.
"""
x = np.random.randint(0, high=500, size=100)
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -584,7 +584,7 @@ class TestPythonBinding(unittest.TestCase):
Test a row vector input parameter.
"""
x = np.random.rand(100)
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -620,7 +620,7 @@ class TestPythonBinding(unittest.TestCase):
Test an unsigned row vector input parameter.
"""
x = np.random.randint(0, high=500, size=100)
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -656,7 +656,7 @@ class TestPythonBinding(unittest.TestCase):
Test that we can pass a matrix with all numeric features.
"""
x = np.random.rand(100, 10)
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -696,7 +696,7 @@ class TestPythonBinding(unittest.TestCase):
x = pd.DataFrame(np.random.rand(10, 4), columns=list('abcd'))
x['e'] = pd.Series(['a', 'b', 'c', 'd', 'a', 'b', 'e', 'c', 'a', 'b'],
dtype='category')
z = copy.copy(x)
z = x.copy(deep=True)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -787,12 +787,12 @@ class TestPythonBinding(unittest.TestCase):
self.assertEqual(output2['model_bw_out'], 20.0)
def testOneDimensionNumpymatrix(self):
def testOneDimensionNumpyMatrix(self):
"""
Test that we can pass one dimension matrix from matrix_in
"""
x = np.random.rand(100)
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -829,7 +829,7 @@ class TestPythonBinding(unittest.TestCase):
Same as testNumpyMatrix() but with an unsigned matrix and One Dimension Matrix.
"""
x = np.random.randint(0, high=500, size=100)
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -865,7 +865,7 @@ class TestPythonBinding(unittest.TestCase):
Test that we pass Two Dimension column vetor as input paramter
"""
x = np.random.rand(100,1)
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -901,7 +901,7 @@ class TestPythonBinding(unittest.TestCase):
Test that we pass Two Dimension unsigned column vector input parameter.
"""
x = np.random.randint(0, high=500, size=[100, 1])
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -935,7 +935,7 @@ class TestPythonBinding(unittest.TestCase):
Test a two dimensional row vector input parameter.
"""
x = np.random.rand(100,1)
z =copy.copy(x)
z =copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -971,7 +971,7 @@ class TestPythonBinding(unittest.TestCase):
Test an unsigned two dimensional row vector input parameter.
"""
x = np.random.randint(0, high=500, size=[100, 1])
z = copy.copy(x)
z = copy.deepcopy(x)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -1007,7 +1007,7 @@ class TestPythonBinding(unittest.TestCase):
Test that we can pass a one dimension matrix with some categorical features.
"""
x = pd.DataFrame(np.random.rand(10))
z = copy.copy(x)
z = x.copy(deep=True)
output = test_python_binding(string_in='hello',
int_in=12,
@@ -1017,7 +1017,7 @@ class TestPythonBinding(unittest.TestCase):
self.assertEqual(output['matrix_and_info_out'].shape[0], 10)
for i in range(10):
self.assertEqual(output['matrix_and_info_out'][i, 0], z[0][i] * 2)
self.assertEqual(output['matrix_and_info_out'][i, 0], x[0][i] * 2)
def testOneDimensionMatrixAndInfoPandasForceCopy(self):
"""