From c6a0fe1f14aa144c0f2d5236586d2eca0486bb6a Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Sat, 16 Nov 2019 16:19:42 +0100 Subject: [PATCH] Use the correct steps path and use variables for the windows steps. --- .ci/ci.yaml | 29 +++- .ci/linux-steps.yaml | 5 +- .ci/macos-steps.yaml | 5 +- ...ows-vs14-steps.yaml => windows-steps.yaml} | 27 ++-- .ci/windows-vs15-steps.yaml | 131 ------------------ .ci/windows-vs16-steps.yaml | 130 ----------------- 6 files changed, 43 insertions(+), 284 deletions(-) rename .ci/{windows-vs14-steps.yaml => windows-steps.yaml} (87%) delete mode 100644 .ci/windows-vs15-steps.yaml delete mode 100644 .ci/windows-vs16-steps.yaml diff --git a/.ci/ci.yaml b/.ci/ci.yaml index 6fb559cb75..9656ce1dd3 100644 --- a/.ci/ci.yaml +++ b/.ci/ci.yaml @@ -18,7 +18,7 @@ jobs: CMakeArgs: '-DDEBUG=OFF -DPROFILE=OFF -DBUILD_MARKDOWN_BINDINGS=ON -DBUILD_PYTHON_BINDINGS=OFF' steps: - - template: ci/linux-steps.yaml + - template: linux-steps.yaml - job: macOS pool: @@ -36,22 +36,28 @@ jobs: CMakeArgs: '-DDEBUG=OFF -DPROFILE=OFF' steps: - - template: ci/macos-steps.yaml + - template: macos-steps.yaml - job: WindowsVS14 timeoutInMinutes: 360 + displayName: Windows VS14 pool: vmImage: vs2015-win2012r2 strategy: matrix: Plain: CMakeArgs: '-DDEBUG=ON -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF' - + CMakeGenerator: '-G "Visual Studio 14 2015 Win64"' + MSBuildVersion: '14.0' + ArchiveNoLibs: 'mlpack-windows-vs14-no-libs.zip' + ArchiveLibs: 'mlpack-windows-vs14.zip' + ArchiveTests: 'mlpack_test-vs14.xml' steps: - - template: ci/windows-vs14-steps.yaml + - template: windows-steps.yaml - job: WindowsVS15 timeoutInMinutes: 360 + displayName: Windows VS15 pool: vmImage: vs2017-win2016 strategy: @@ -59,12 +65,18 @@ jobs: Plain: CMakeArgs: '-DDEBUG=ON -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF' python.version: '2.7' + CMakeGenerator: '-G "Visual Studio 15 2017 Win64"' + MSBuildVersion: '15.0' + ArchiveNoLibs: 'mlpack-windows-vs15-no-libs.zip' + ArchiveLibs: 'mlpack-windows-vs15.zip' + ArchiveTests: 'mlpack_test-vs15.xml' steps: - - template: ci/windows-vs15-steps.yaml + - template: windows-steps.yaml - job: WindowsVS16 timeoutInMinutes: 360 + displayName: Windows VS16 pool: vmImage: windows-2019 strategy: @@ -72,6 +84,11 @@ jobs: Plain: CMakeArgs: '-DDEBUG=ON -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF' python.version: '2.7' + CMakeGenerator: '-G "Visual Studio 16 2019"' + MSBuildVersion: '16.0' + ArchiveNoLibs: 'mlpack-windows-vs16-no-libs.zip' + ArchiveLibs: 'mlpack-windows-vs16.zip' + ArchiveTests: 'mlpack_test-vs16.xml' steps: - - template: ci/windows-vs16-steps.yaml \ No newline at end of file + - template: windows-steps.yaml diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index 914924a07a..e94464013f 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -11,6 +11,8 @@ steps: # Install build dependencies - script: | + git clone --depth 1 https://github.com/mlpack/jenkins-conf.git conf + sudo apt-get update unset BOOST_ROOT sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost-math-dev libboost-program-options-dev libboost-test-dev libboost-serialization-dev libarmadillo-dev xz-utils @@ -45,8 +47,7 @@ steps: # Process test results - bash: | - curl https://gist.githubusercontent.com/zoq/9ccf753c4b61efcc5e6f0452c9c22e26/raw/a7790189a530925a5575b2dc45fdc6b915b01313/convert.py -O - python convert.py > mlpack_test.xml + python conf/ci/convert.py > mlpack_test.xml displayName: 'Process test results' # Publish test results to Azure Pipelines diff --git a/.ci/macos-steps.yaml b/.ci/macos-steps.yaml index 8e697e840d..e60c3483dc 100644 --- a/.ci/macos-steps.yaml +++ b/.ci/macos-steps.yaml @@ -16,6 +16,8 @@ steps: unset BOOST_ROOT pip install cython numpy pandas brew install openblas armadillo boost + + git clone --depth 1 https://github.com/mlpack/jenkins-conf.git conf displayName: 'Install Build Dependencies' # Configure mlpack (CMake) @@ -36,8 +38,7 @@ steps: # Process test results - bash: | - curl https://gist.githubusercontent.com/zoq/9ccf753c4b61efcc5e6f0452c9c22e26/raw/a7790189a530925a5575b2dc45fdc6b915b01313/convert.py -O - python convert.py > mlpack_test.xml + python conf/ci/convert.py > mlpack_test.xml displayName: 'Process test results' # Publish test results to Azure Pipelines diff --git a/.ci/windows-vs14-steps.yaml b/.ci/windows-steps.yaml similarity index 87% rename from .ci/windows-vs14-steps.yaml rename to .ci/windows-steps.yaml index a940a7fc6b..d9561ae830 100644 --- a/.ci/windows-vs14-steps.yaml +++ b/.ci/windows-steps.yaml @@ -27,9 +27,11 @@ steps: # Configure armadillo - bash: | + git clone --depth 1 https://github.com/mlpack/jenkins-conf.git conf + curl http://masterblaster.mlpack.org:5005/armadillo-8.400.0.tar.gz | tar xvz - cd armadillo-8.400.0/ && cmake -G "Visual Studio 14 2015 Win64" \ + cd armadillo-8.400.0/ && cmake $(CMakeGenerator) \ -DBLAS_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ -DLAPACK_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ -DCMAKE_PREFIX:FILEPATH=../../armadillo \ @@ -42,7 +44,7 @@ steps: inputs: solution: 'armadillo-8.400.0/*.sln' msbuildLocationMethod: 'location' - msbuildVersion: '14.0' + msbuildVersion: $(MSBuildVersion) configuration: 'Release' msbuildArchitecture: 'x64' msbuildArguments: /m /p:BuildInParallel=true @@ -55,7 +57,7 @@ steps: mkdir build cd build - cmake -G "Visual Studio 14 2015 Win64" ` + cmake $(CMakeGenerator) ` $(CMakeArgs) ` -DBLAS_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a ` -DLAPACK_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a ` @@ -71,7 +73,7 @@ steps: inputs: solution: 'build/*.sln' msbuildLocationMethod: 'location' - msbuildVersion: '15.0' + msbuildVersion: $(MSBuildVersion) configuration: 'Release' msbuildArchitecture: 'x64' msbuildArguments: /m /p:BuildInParallel=true @@ -95,8 +97,7 @@ steps: # Process test results - bash: | - curl https://gist.githubusercontent.com/zoq/9ccf753c4b61efcc5e6f0452c9c22e26/raw/a7790189a530925a5575b2dc45fdc6b915b01313/convert.py -O - python convert.py > mlpack_test.xml + python conf/ci/convert.py > mlpack_test.xml displayName: 'Process test results' # Copy artifacts @@ -111,7 +112,7 @@ steps: rootFolderOrFile: 'exe-archive\' includeRootFolder: false archiveType: 'zip' - archiveFile: 'mlpack-windows-vs14-no-libs.zip' + archiveFile: $(ArchiveNoLibs) replaceExistingArchive: true displayName: 'Build artifacts' @@ -120,25 +121,25 @@ steps: rootFolderOrFile: 'build\Release\' includeRootFolder: false archiveType: 'zip' - archiveFile: 'mlpack-windows-vs14.zip' + archiveFile: $(ArchiveLibs) replaceExistingArchive: true displayName: 'Build artifacts' # Publish artifacts to Azure Pipelines - task: PublishBuildArtifacts@1 inputs: - pathtoPublish: 'mlpack-windows-vs14-no-libs.zip' - artifactName: 'mlpack-windows-vs14-no-libs.zip' + pathtoPublish: $(ArchiveNoLibs) + artifactName: $(ArchiveNoLibs) displayName: 'Publish artifacts no-libs' - task: PublishBuildArtifacts@1 inputs: - pathtoPublish: 'mlpack-windows-vs14.zip' - artifactName: 'mlpack-windows-vs14.zip' + pathtoPublish: $(ArchiveLibs) + artifactName: $(ArchiveLibs) displayName: 'Publish artifacts complete' - task: PublishBuildArtifacts@1 inputs: pathtoPublish: 'mlpack_test.xml' - artifactName: 'mlpack_test-vs14.xml' + artifactName: $(ArchiveTests) displayName: 'Publish artifacts test results' # Publish test results to Azure Pipelines diff --git a/.ci/windows-vs15-steps.yaml b/.ci/windows-vs15-steps.yaml deleted file mode 100644 index b2e6ef628a..0000000000 --- a/.ci/windows-vs15-steps.yaml +++ /dev/null @@ -1,131 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 1 -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' -- task: NuGetToolInstaller@0 - inputs: - versionSpec: '5.3.0' - -# Fetch build dependencies -- bash: | - nuget install OpenBLAS -o $(Agent.ToolsDirectory) - nuget install boost -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_unit_test_framework-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_program_options-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_random-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_serialization-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_math_c99-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install OpenBLAS -o $(Agent.ToolsDirectory) - - mkdir -p $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_program_options-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_math_c99-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_random-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_serialization-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_unit_test_framework-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - - curl http://masterblaster.mlpack.org:5005/armadillo-8.400.0.tar.gz | tar xvz - displayName: 'Fetch build dependencies' - -# Configure armadillo -- bash: | - cd armadillo-8.400.0/ && cmake -G "Visual Studio 15 2017 Win64" \ - -DBLAS_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ - -DLAPACK_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ - -DCMAKE_PREFIX:FILEPATH=../../armadillo \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_BUILD_TYPE=Release . - displayName: 'Configure armadillo' - -# Build armadillo -- task: MSBuild@1 - inputs: - solution: 'armadillo-8.400.0/*.sln' - msbuildLocationMethod: 'location' - msbuildVersion: '15.0' - configuration: 'Release' - msbuildArchitecture: 'x64' - msbuildArguments: /m /p:BuildInParallel=true - clean: false - maximumCpuCount: false - displayName: 'Build armadillo' - -# Configure mlpack -- bash: | - mkdir build && cd build && \ - cmake -G "Visual Studio 15 2017 Win64" \ - -DBLAS_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ - -DLAPACK_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ - -DARMADILLO_INCLUDE_DIR="../armadillo-8.400.0/include" \ - -DARMADILLO_LIBRARY:FILEPATH="../armadillo-8.400.0/Release/armadillo.lib" \ - -DBOOST_INCLUDEDIR:PATH=$(Agent.ToolsDirectory)/boost.1.60.0.0/lib/native/include \ - -DBOOST_LIBRARYDIR:PATH=$(Agent.ToolsDirectory)/boost_libs \ - -DDEBUG=OFF \ - -DPROFILE=OFF \ - -DBUILD_PYTHON_BINDINGS=OFF \ - -DCMAKE_BUILD_TYPE=Release .. \ - displayName: 'Configure mlpack' - -# Build mlpack -- task: MSBuild@1 - inputs: - solution: 'build/*.sln' - msbuildLocationMethod: 'location' - msbuildVersion: '15.0' - configuration: 'Release' - msbuildArchitecture: 'x64' - msbuildArguments: /m /p:BuildInParallel=true - maximumCpuCount: false - clean: false - displayName: 'Build mlpack' - -# Run tests -- powershell: | - cp $(Agent.ToolsDirectory)\boost_libs\*.* build\ - cp $(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib/x64\*.* build\ - cp $(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* build\ - - cd build - Release\mlpack_test.exe --result_code=no --report_level=detailed --log_level=test_suite --log_format=HRF --log_sink=res.txt - displayName: 'Run tests' - -# Process test results -- bash: | - curl https://gist.githubusercontent.com/zoq/9ccf753c4b61efcc5e6f0452c9c22e26/raw/a7790189a530925a5575b2dc45fdc6b915b01313/convert.py -O - python convert.py > mlpack_test.xml - displayName: 'Process test results' - -# Publish test results to Azure Pipelines -- task: PublishTestResults@2 - inputs: - testResultsFormat: JUnit - testResultsFiles: 'mlpack_test.xml' - failTaskOnFailedTests: true - displayName: 'Publish tests' - -# Build artifacts -- bash: | - 7z a mlpack-windows-vs15-no-libs.zip build/Release/*.exe - 7z a mlpack-windows-vs15.zip build/Release/*.* - displayName: 'Build artifacts' - -# Publish build artifacts to Azure Pipelines -- task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: 'mlpack-windows-vs15-no-libs.zip' - artifactName: 'mlpack-windows-vs15-no-libs.zip' - displayName: 'Publish artifacts no-libs' -- task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: 'mlpack-windows-vs15.zip' - artifactName: 'mlpack-windows-vs15.zip' - displayName: 'Publish artifacts complete' -- task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: 'mlpack_test.xml' - artifactName: 'mlpack_test_vs15.xml' - displayName: 'Publish artifacts test results' - diff --git a/.ci/windows-vs16-steps.yaml b/.ci/windows-vs16-steps.yaml deleted file mode 100644 index 87c23251f9..0000000000 --- a/.ci/windows-vs16-steps.yaml +++ /dev/null @@ -1,130 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 1 -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' -- task: NuGetToolInstaller@0 - inputs: - versionSpec: '5.3.0' - -# Fetch build dependencies -- bash: | - nuget install OpenBLAS -o $(Agent.ToolsDirectory) - nuget install boost -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_unit_test_framework-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_program_options-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_random-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_serialization-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install boost_math_c99-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0 - nuget install OpenBLAS -o $(Agent.ToolsDirectory) - - mkdir -p $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_program_options-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_math_c99-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_random-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_serialization-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - cp $(Agent.ToolsDirectory)/boost_unit_test_framework-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs - - curl http://masterblaster.mlpack.org:5005/armadillo-8.400.0.tar.gz | tar xvz - displayName: 'Fetch build dependencies' - -# Configure armadillo -- bash: | - cd armadillo-8.400.0/ && cmake -G "Visual Studio 16 2019" \ - -DBLAS_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ - -DLAPACK_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ - -DCMAKE_PREFIX:FILEPATH=../../armadillo \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_BUILD_TYPE=Release . - displayName: 'Configure armadillo' - -# Build armadillo -- task: MSBuild@1 - inputs: - solution: 'armadillo-8.400.0/*.sln' - msbuildLocationMethod: 'location' - msbuildVersion: '16.0' - configuration: 'Release' - msbuildArchitecture: 'x64' - msbuildArguments: /m /p:BuildInParallel=true - clean: false - maximumCpuCount: false - displayName: 'Build armadillo' - -# Configure mlpack -- bash: | - mkdir build && cd build && \ - cmake -G "Visual Studio 16 2019" \ - -DBLAS_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ - -DLAPACK_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \ - -DARMADILLO_INCLUDE_DIR="../armadillo-8.400.0/include" \ - -DARMADILLO_LIBRARY:FILEPATH="../armadillo-8.400.0/Release/armadillo.lib" \ - -DBOOST_INCLUDEDIR:PATH=$(Agent.ToolsDirectory)/boost.1.60.0.0/lib/native/include \ - -DBOOST_LIBRARYDIR:PATH=$(Agent.ToolsDirectory)/boost_libs \ - -DDEBUG=OFF \ - -DPROFILE=OFF \ - -DBUILD_PYTHON_BINDINGS=OFF \ - -DCMAKE_BUILD_TYPE=Release .. \ - displayName: 'Configure mlpack' - -# Build mlpack -- task: MSBuild@1 - inputs: - solution: 'build/*.sln' - msbuildLocationMethod: 'location' - msbuildVersion: '15.0' - configuration: 'Release' - msbuildArchitecture: 'x64' - msbuildArguments: /m /p:BuildInParallel=true - maximumCpuCount: false - clean: false - displayName: 'Build mlpack' - -# Run tests -- powershell: | - cp $(Agent.ToolsDirectory)\boost_libs\*.* build\ - cp $(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib/x64\*.* build\ - cp $(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* build\ - - cd build - Release\mlpack_test.exe --result_code=no --report_level=detailed --log_level=test_suite --log_format=HRF --log_sink=res.txt - displayName: 'Run tests' - -# Process test results -- bash: | - curl https://gist.githubusercontent.com/zoq/9ccf753c4b61efcc5e6f0452c9c22e26/raw/a7790189a530925a5575b2dc45fdc6b915b01313/convert.py -O - python convert.py > mlpack_test.xml - displayName: 'Process test results' - -# Publish test results to Azure Pipelines -- task: PublishTestResults@2 - inputs: - testResultsFormat: JUnit - testResultsFiles: 'mlpack_test.xml' - failTaskOnFailedTests: true - displayName: 'Publish tests' - -# Build artifacts -- bash: | - 7z a mlpack-windows-vs16-no-libs.zip build/Release/*.exe - 7z a mlpack-windows-vs16.zip build/Release/*.* - displayName: 'Build artifacts' - -# Publish build artifacts to Azure Pipelines -- task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: 'mlpack-windows-vs16-no-libs.zip' - artifactName: 'mlpack-windows-vs16-no-libs.zip' - displayName: 'Publish artifacts no-libs' -- task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: 'mlpack-windows-vs16.zip' - artifactName: 'mlpack-windows-vs16.zip' - displayName: 'Publish artifacts complete' -- task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: 'mlpack_test.xml' - artifactName: 'mlpack_test_vs16.xml' - displayName: 'Publish artifacts test results'