CMake/CI: Cache the CUDA test builds with ccache
libeigen/eigen!2874 Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com> Co-authored-by: Rasmus Munk Larsen <rlarsen@nvidia.com>
This commit is contained in:
Rasmus Munk Larsen
co-authored by
Rasmus Munk Larsen
Rasmus Munk Larsen
Rasmus Munk Larsen
Rasmus Munk Larsen
parent
16e143a970
commit
12c9c14bc5
@@ -121,6 +121,10 @@ opts out of Eigen's install rules. They exist because those are claims
|
||||
[`doc/TopicCMakeGuide.dox`](../doc/TopicCMakeGuide.dox) makes to users and nothing else checks; the blocking
|
||||
documentation job only builds the docs, it does not run what they describe.
|
||||
|
||||
Not every scenario answers to the documentation. A find module that has to survive a second configure of the same
|
||||
build tree, or the wiring that routes a compiler launcher into a test's compile command, is CMake behavior nothing
|
||||
else exercises either.
|
||||
|
||||
```bash
|
||||
cmake -G Ninja -S . -B build -DEIGEN_BUILD_TESTING=ON
|
||||
cmake -E chdir build ctest -L buildsystem --output-on-failure
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
EIGEN_CI_BUILD_TARGET: buildtests
|
||||
EIGEN_CI_SKIP_APT: "false"
|
||||
# "off" disables the ccache compiler launchers (see build.linux.script.sh);
|
||||
# used where the dominant compiles bypass them (nvcc-driven .cu files) or
|
||||
# the compiler is unsupported by ccache (nvc++). Images without ccache
|
||||
# build exactly as before regardless of this setting.
|
||||
# used where the compiler is unsupported by ccache (nvc++) or a cache does
|
||||
# not survive the image (ROCm). Images without ccache build exactly as
|
||||
# before regardless of this setting.
|
||||
EIGEN_CI_CCACHE: "on"
|
||||
CCACHE_DIR: ${CI_PROJECT_DIR}/.ccache
|
||||
# A full buildtests job stores ~4-6 GB of compressed objects; at 2G every
|
||||
@@ -258,12 +258,13 @@ build:linux:x86-64:nvhpc-26.1:default:unsupported:
|
||||
|
||||
.build:linux:cuda:
|
||||
extends: .build:linux:cross:x86-64
|
||||
# No caching: the .cu compiles that dominate these builds go through nvcc,
|
||||
# which bypasses the C/CXX ccache launchers, and the CUDA images vary
|
||||
# enough between runs that a cache invites staleness.
|
||||
cache: []
|
||||
# ccache and the inherited cache stanza apply here: the .cu compiles that
|
||||
# dominate these builds now reach the launcher through the wrapper that
|
||||
# ei_cuda_use_compiler_launcher() puts in front of nvcc. A new image costs
|
||||
# one cold build rather than serving stale objects, since ccache hashes the
|
||||
# nvcc it runs and the preprocessed source. nvcc's -M dependency and -dlink
|
||||
# passes are not cacheable and appear as misses in the statistics.
|
||||
variables:
|
||||
EIGEN_CI_CCACHE: "off"
|
||||
# Additional flags passed to the cuda compiler.
|
||||
EIGEN_CI_CUDA_CXX_FLAGS: ""
|
||||
# Compute architectures present in the GitLab CI runners.
|
||||
|
||||
@@ -265,9 +265,9 @@ test:linux:x86-64:nvhpc-26.1:default:unsupported:
|
||||
allow_failure: true
|
||||
variables:
|
||||
EIGEN_CI_CTEST_LABEL: gpu
|
||||
# nvcc-driven builds bypass ccache and nvcc-linked binaries are not
|
||||
# bit-reproducible, so the pass cache could never hit here. cache: []
|
||||
# also drops the inherited stanza, mirroring the build jobs' opt-outs.
|
||||
# nvcc-linked binaries are not bit-reproducible, so the content-addressed
|
||||
# pass cache could never hit here. cache: [] also drops the inherited
|
||||
# stanza.
|
||||
EIGEN_CI_TEST_CACHE: "off"
|
||||
cache: []
|
||||
rules: !reference [.rules:libeigen:gpu, rules]
|
||||
|
||||
@@ -16,6 +16,97 @@ if(EIGEN_TEST_HIP AND NOT DEFINED EIGEN_HIP_ARCHITECTURES)
|
||||
CACHE STRING "HIP GPU architectures to build Eigen's HIP tests for.")
|
||||
endif()
|
||||
|
||||
# Renders a command as one line of POSIX shell source that runs it with these
|
||||
# exact argument boundaries. CMAKE_<LANG>_COMPILER_LAUNCHER is a list in which
|
||||
# each element is one argv entry, and an element may itself contain spaces or
|
||||
# characters the shell would act on, so the elements cannot simply be joined.
|
||||
function(ei_quote_command_for_shell out_var)
|
||||
set(quoted "")
|
||||
foreach(arg IN LISTS ARGN)
|
||||
# Single quotes protect every character but a single quote, which is
|
||||
# spliced back in as '\'' -- close, escape, reopen.
|
||||
string(REPLACE "'" "'\\''" arg "${arg}")
|
||||
if(quoted)
|
||||
string(APPEND quoted " ")
|
||||
endif()
|
||||
string(APPEND quoted "'${arg}'")
|
||||
endforeach()
|
||||
set(${out_var} "${quoted}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# The same for one line of cmd.exe batch source.
|
||||
function(ei_quote_command_for_batch out_var)
|
||||
set(quoted "")
|
||||
foreach(arg IN LISTS ARGN)
|
||||
# Double quotes are the only grouping cmd.exe offers, it has no escape for
|
||||
# a literal one, and it expands %VAR% and delayed !VAR! even between them.
|
||||
# Refuse rather than write a wrapper that would run something else.
|
||||
if(arg MATCHES "[\"%!]")
|
||||
message(FATAL_ERROR "cannot quote '${arg}' for cmd.exe: a command line "
|
||||
"argument containing \" % or ! is not representable "
|
||||
"in a batch file")
|
||||
endif()
|
||||
if(quoted)
|
||||
string(APPEND quoted " ")
|
||||
endif()
|
||||
string(APPEND quoted "\"${arg}\"")
|
||||
endforeach()
|
||||
set(${out_var} "${quoted}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# Writes <dir>/eigen-nvcc-launcher.{sh,bat}, which runs <nvcc> under the
|
||||
# launcher argv given in ARGN and forwards its own arguments unchanged, and
|
||||
# returns its path through out_var.
|
||||
function(ei_write_nvcc_launcher_wrapper out_var dir nvcc)
|
||||
if(CMAKE_HOST_WIN32)
|
||||
set(wrapper "${dir}/eigen-nvcc-launcher.bat")
|
||||
ei_quote_command_for_batch(command ${ARGN} "${nvcc}")
|
||||
file(WRITE "${wrapper}" "@echo off\n${command} %*\n")
|
||||
else()
|
||||
set(wrapper "${dir}/eigen-nvcc-launcher.sh")
|
||||
ei_quote_command_for_shell(command ${ARGN} "${nvcc}")
|
||||
file(WRITE "${wrapper}" "#!/bin/sh\nexec ${command} \"$@\"\n")
|
||||
# file(CHMOD) would need CMake 3.19; this project's minimum is 3.17.
|
||||
execute_process(COMMAND chmod +x "${wrapper}")
|
||||
endif()
|
||||
set(${out_var} "${wrapper}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# FindCUDA's cuda_add_executable() bakes CUDA_NVCC_EXECUTABLE into a generated
|
||||
# run_nvcc.cmake and runs it as a quoted `COMMAND "${CUDA_NVCC_EXECUTABLE}"`, so
|
||||
# it never consults CMAKE_CUDA_COMPILER_LAUNCHER -- and a launcher list such as
|
||||
# "ccache;nvcc" cannot be substituted either, because the COMMAND is one quoted
|
||||
# argument. A wrapper script is therefore the only available hook. Without it,
|
||||
# configuring ccache or sccache speeds up the C++ tests while silently skipping
|
||||
# every .cu translation unit, which are the slowest in the tree.
|
||||
#
|
||||
# Only the nvcc path needs this: EIGEN_TEST_CUDA_CLANG and EIGEN_TEST_CUDA_NVC
|
||||
# compile .cu as CXX and already pick up CMAKE_CXX_COMPILER_LAUNCHER.
|
||||
#
|
||||
# This can be deleted once the CUDA language is enabled directly, i.e. when
|
||||
# CMP0146 (see the top-level CMakeLists.txt) no longer has to be set to OLD.
|
||||
#
|
||||
# Caveat: the wrapper also fronts nvcc's -M dependency and -dlink passes, which
|
||||
# ccache does not cache; expect those as misses in the statistics.
|
||||
macro(ei_cuda_use_compiler_launcher)
|
||||
# Fall back to the C++ launcher. A project using FindCUDA never enables the
|
||||
# CUDA language, so CMAKE_CUDA_COMPILER_LAUNCHER is seldom set, whereas
|
||||
# CMAKE_CXX_COMPILER_LAUNCHER usually is -- including in Eigen's own CI.
|
||||
set(EIGEN_NVCC_LAUNCHER "${CMAKE_CUDA_COMPILER_LAUNCHER}")
|
||||
if(NOT EIGEN_NVCC_LAUNCHER)
|
||||
set(EIGEN_NVCC_LAUNCHER "${CMAKE_CXX_COMPILER_LAUNCHER}")
|
||||
endif()
|
||||
# The MATCHES guard makes this a one-time setup per directory: the set() below
|
||||
# shadows the cache entry for the rest of this scope, and we are called once
|
||||
# per test.
|
||||
if(EIGEN_NVCC_LAUNCHER AND NOT CUDA_NVCC_EXECUTABLE MATCHES "eigen-nvcc-launcher")
|
||||
ei_write_nvcc_launcher_wrapper(EIGEN_NVCC_WRAPPER "${CMAKE_CURRENT_BINARY_DIR}"
|
||||
"${CUDA_NVCC_EXECUTABLE}" ${EIGEN_NVCC_LAUNCHER})
|
||||
set(CUDA_NVCC_EXECUTABLE "${EIGEN_NVCC_WRAPPER}")
|
||||
message(STATUS "CUDA tests: nvcc routed through compiler launcher '${EIGEN_NVCC_LAUNCHER}'")
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
#internal. See documentation of ei_add_test for details.
|
||||
macro(ei_add_test_internal testname testname_with_suffix)
|
||||
set(targetname ${testname_with_suffix})
|
||||
@@ -70,6 +161,7 @@ macro(ei_add_test_internal testname testname_with_suffix)
|
||||
endif()
|
||||
target_link_libraries(${targetname} ${CUDA_NVC_LINK_LIBRARIES})
|
||||
else()
|
||||
ei_cuda_use_compiler_launcher()
|
||||
cuda_add_executable(${targetname} ${filename})
|
||||
endif()
|
||||
else()
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# SPDX-FileCopyrightText: The Eigen Authors
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
# Build-system integration tests. Each scenario drives a real configure,
|
||||
# build, and install of Eigen through CMake and asserts a claim that
|
||||
# doc/TopicCMakeGuide.dox makes about consuming Eigen, so documented CMake
|
||||
# behavior cannot drift from the tree without a test failing.
|
||||
# Build-system integration tests. Each scenario asserts a claim about Eigen's
|
||||
# own CMake surface -- most by driving a real configure, build, and install of
|
||||
# Eigen -- so documented or relied-upon CMake behavior cannot drift from the
|
||||
# tree without a test failing.
|
||||
#
|
||||
# Registered only for a top-level, non-cross-compiling build: a scenario
|
||||
# configures and runs a nested project, which needs a usable host toolchain
|
||||
@@ -22,6 +22,7 @@ set(EIGEN_BUILDSYSTEM_SCENARIOS
|
||||
install_off
|
||||
install_off_all_options
|
||||
exclude_from_all
|
||||
cuda_launcher_wrapper
|
||||
)
|
||||
|
||||
# find_package version ranges are CMake 3.19 syntax. On the 3.17 Eigen still
|
||||
|
||||
@@ -51,6 +51,12 @@ function(bs_fail message)
|
||||
message(FATAL_ERROR "[${SCENARIO}] ${message}")
|
||||
endfunction()
|
||||
|
||||
function(bs_assert_streq actual expected what)
|
||||
if(NOT actual STREQUAL expected)
|
||||
bs_fail("${what}:\n expected [${expected}]\n actual [${actual}]")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
# Runs a command, failing the scenario with its merged output unless the exit
|
||||
# status matches EXPECT_RESULT. EXPECT_RESULT accepts a number or the word
|
||||
# FAILURE, which admits any non-zero status.
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-FileCopyrightText: The Eigen Authors
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
# ei_cuda_use_compiler_launcher() is the only way a compiler launcher reaches
|
||||
# nvcc, because FindCUDA's cuda_add_executable() runs CUDA_NVCC_EXECUTABLE as a
|
||||
# single quoted COMMAND. CMAKE_<LANG>_COMPILER_LAUNCHER is a list whose
|
||||
# elements are argv entries, and an element may contain spaces, so the wrapper
|
||||
# has to preserve argument boundaries rather than flatten the list into one
|
||||
# line of shell source.
|
||||
#
|
||||
# No CUDA installation is involved: the launcher and nvcc are stand-ins that
|
||||
# report the argv they were handed.
|
||||
|
||||
include("${EIGEN_SOURCE_DIR}/cmake/EigenTesting.cmake")
|
||||
|
||||
# A cmd.exe wrapper cannot be executed from a POSIX host, so its quoting rules
|
||||
# are pinned on the rendering instead, on every host.
|
||||
ei_quote_command_for_batch(rendered "C:/a dir/ccache.exe" "--opt=x y" "C:/cuda/nvcc.exe")
|
||||
bs_assert_streq("${rendered}"
|
||||
"\"C:/a dir/ccache.exe\" \"--opt=x y\" \"C:/cuda/nvcc.exe\""
|
||||
"cmd.exe rendering of a launcher command")
|
||||
|
||||
if(CMAKE_HOST_WIN32)
|
||||
return()
|
||||
endif()
|
||||
|
||||
# Put the stand-ins under a directory whose name contains a space, so the paths
|
||||
# the wrapper bakes in are quoted too, not only the launcher's own arguments.
|
||||
set(bin_dir "${WORK_DIR}/fake bin")
|
||||
file(MAKE_DIRECTORY "${bin_dir}")
|
||||
|
||||
set(fake_nvcc "${bin_dir}/fake nvcc")
|
||||
file(WRITE "${fake_nvcc}" "#!/bin/sh\nfor a in \"$@\"; do echo \"nvcc-arg=[$a]\"; done\n")
|
||||
execute_process(COMMAND chmod +x "${fake_nvcc}")
|
||||
|
||||
# Reports the one launcher argument it expects, then runs what follows it --
|
||||
# which is how ccache is invoked, and what proves nvcc is still a separate argv
|
||||
# entry rather than part of the launcher's.
|
||||
set(fake_launcher "${bin_dir}/fake launcher")
|
||||
file(WRITE "${fake_launcher}" "#!/bin/sh\necho \"launcher-arg=[$1]\"\nshift\nexec \"$@\"\n")
|
||||
execute_process(COMMAND chmod +x "${fake_launcher}")
|
||||
|
||||
set(CMAKE_CXX_COMPILER_LAUNCHER "${fake_launcher}" "--launcher opt")
|
||||
set(CUDA_NVCC_EXECUTABLE "${fake_nvcc}")
|
||||
set(CMAKE_CURRENT_BINARY_DIR "${WORK_DIR}")
|
||||
ei_cuda_use_compiler_launcher()
|
||||
|
||||
if(CUDA_NVCC_EXECUTABLE STREQUAL "${fake_nvcc}")
|
||||
bs_fail("ei_cuda_use_compiler_launcher() left CUDA_NVCC_EXECUTABLE unwrapped")
|
||||
endif()
|
||||
|
||||
bs_run(WHAT "generated nvcc wrapper" OUTPUT_VARIABLE output
|
||||
COMMAND "${CUDA_NVCC_EXECUTABLE}" "-Ia dir" "-DQ=\"a b\"" "plain")
|
||||
|
||||
foreach(expected "launcher-arg=[--launcher opt]"
|
||||
"nvcc-arg=[-Ia dir]"
|
||||
"nvcc-arg=[-DQ=\"a b\"]"
|
||||
"nvcc-arg=[plain]")
|
||||
string(FIND "${output}" "${expected}" position)
|
||||
if(position LESS 0)
|
||||
bs_fail("the wrapper did not pass through ${expected}\n----\n${output}\n----")
|
||||
endif()
|
||||
endforeach()
|
||||
Reference in New Issue
Block a user