From 9be565330fab04c7e47c2d3be33daefed5fdf300 Mon Sep 17 00:00:00 2001 From: Rasmus Munk Larsen <4643818-rmlarsen1@users.noreply.gitlab.com> Date: Thu, 20 Aug 2026 15:45:06 -0700 Subject: [PATCH] CI: Add an affected-tests tier between smoke and all-tests libeigen/eigen!2756 Co-authored-by: Rasmus Munk Larsen Co-authored-by: Rasmus Munk Larsen --- .agents/ci.md | 102 +++++- .agents/testing.md | 4 +- .gitlab-ci.yml | 6 + ci/build.linux.gitlab-ci.yml | 75 +++++ ci/common.gitlab-ci.yml | 85 +++++ ci/scripts/build.linux.script.sh | 130 +++++++- ci/scripts/test.linux.script.sh | 31 ++ ci/select.gitlab-ci.yml | 34 ++ ci/test.linux.gitlab-ci.yml | 60 ++++ cmake/EigenTesting.cmake | 9 + scripts/affected_tests.py | 533 +++++++++++++++++++++++++++++ scripts/test_affected_tests.py | 554 +++++++++++++++++++++++++++++++ 12 files changed, 1613 insertions(+), 10 deletions(-) create mode 100644 ci/select.gitlab-ci.yml create mode 100644 scripts/affected_tests.py create mode 100644 scripts/test_affected_tests.py diff --git a/.agents/ci.md b/.agents/ci.md index be388507e..4b6e83ae3 100644 --- a/.agents/ci.md +++ b/.agents/ci.md @@ -2,8 +2,9 @@ Use the checked-out configuration as the source of truth. [`.gitlab-ci.yml`](../.gitlab-ci.yml) defines stages and includes; [`ci/*.gitlab-ci.yml`](../ci) and [`ci/scripts/`](../ci/scripts) define the actual jobs. Default MR pipelines -run a limited smoke matrix; labels such as `all-tests` and `gpu-tests`, plus scheduled or manually started pipelines, -enable broader jobs. A green default MR pipeline is not proof that every supported configuration was exercised. +run a limited smoke matrix; labels such as `affected-tests`, `all-tests` and `gpu-tests`, plus scheduled or manually +started pipelines, enable broader jobs. A green default MR pipeline is not proof that every supported configuration was +exercised. A pipeline is evidence only for the commit it ran on: after a push, amend, or rebase, check which SHA the pipeline and the merge request point at before citing either — a green run on a superseded revision proves nothing about the @@ -23,6 +24,103 @@ their full selection (fresh clock-derived RNG seeds are part of their coverage), `EIGEN_CI_TEST_CACHE: "off"` opts a job out. Skipped tests are absent from that run's JUnit report, and a test job whose binaries all match cached passes legitimately reports "No tests were found". +## Test Tiers On Merge Requests + +Three tiers, in increasing cost: + +| Tier | Trigger | What runs | +|---|---|---| +| smoke | every MR | the fixed list in [`cmake/EigenSmokeTestList.cmake`](../cmake/EigenSmokeTestList.cmake), usually one part per test, at baseline ISA on x86-64, aarch64 and riscv64 | +| affected | `affected-tests` label | every test the diff can reach, all parts, on x86-64 AVX2 and aarch64, plus the ISA of any packet-math backend the diff touches | +| full | `all-tests` label | the whole suite across the entire compiler and ISA matrix | + +The affected tier exists because the smoke list samples: it is broad but shallow, so a change confined to one module +gets only the one part of each related test that the list happens to name. Reach for `affected-tests` when a change is +module-local and you want depth without paying for the full matrix. + +[`scripts/affected_tests.py`](../scripts/affected_tests.py) computes the selection in the `select:tests` job and writes +`affected/targets.txt` and `affected/ctest_regex.txt`, which the paired build and test jobs consume through +`EIGEN_CI_BUILD_TARGET_FILE` and `EIGEN_CI_CTEST_REGEX_FILE`. Run it locally the same way CI does: + +```bash +python3 scripts/affected_tests.py --base-sha $(git merge-base origin/master HEAD) +python3 scripts/test_affected_tests.py # unit tests, also run by the CI job +``` + +Selection follows the textual `#include` graph, ignoring preprocessor guards, so it is a strict superset of the real +compile dependency and never drops an affected test. Because Eigen is header-only and the umbrella headers are hubs, +a change under `Eigen/src/Core` typically reaches every test and the selector degrades to the full suite — that is the +correct answer, not a failure. Changes to CMake, `ci/`, or the BLAS/LAPACK shims also force the full suite, since they +invalidate the mapping itself. Git rename detection is disabled for the input diff so both the old and new path of a +move are evaluated; an old path absent from the current graph safely forces the full suite. + +The selector derives source-to-target mappings from test CMake registration, including multi-translation-unit +executables and the GPU tests, whose sources are `.cu` because `ei_add_test` takes the extension from +`EIGEN_ADD_TEST_FILENAME_EXTENSION`. A changed test source without a registration is an error rather than an +unconfigured target to drop. `test/buildsystem/` is skipped: its consumers are separate CMake projects that only +`test:linux:buildsystem` configures, so an `add_executable` there is not a registration and its sources reach no +test here. Targets absent from one configuration (optional dependencies such as CHOLMOD, CUDA or +SYCL) are still filtered against `ninja -t targets` after cmake configure, because ninja aborts on an unknown target; +a selection consisting only of such targets is a no-op, not a failure. A missing selection artifact must also fail the +job rather than fall through to the default target, which would silently build everything. + +The build script expands the surviving selection through ninja's phony edges before it shuffles and batches. Most +selected names are aggregates — `buildtests`, and the parent of every split test — and the batch loop can only spread +apart what it is handed, so an unexpanded parent would put a whole test family in one batch and undo the +memory-pressure protection the batching exists for. + +Two registrations do not reduce to a build target. `buildtests` aggregates the `ei_add_test` targets only, so a bare +`add_executable` such as the `bug1213` link regression is named explicitly alongside `buildtests` in the full-suite +mode. The compile-failure suite under `failtest/` is `EXCLUDE_FROM_ALL` and each of its CTest tests builds its own +target as the test action, so those are selected as `_ok` and `_ko` CTest names and never handed to the +build job. Both matter because a `-R` filter silently drops whatever it does not name, while the unfiltered runs in +the other tiers pick them up for free. + +Because that test action is a build in the shared binary directory, `ei_add_failtest` puts the whole suite behind one +`RESOURCE_LOCK`. Without it, `ctest --parallel` starts dozens of concurrent builds over one build system and they +collide whenever a regeneration is pending. The failure is not only noisy: `_ko` is `WILL_FAIL`, so a build system +that errors for an unrelated reason satisfies it just as well as the compile error it is supposed to assert. + +### Backend-Triggered Configurations + +Every job in the default smoke matrix builds at baseline ISA, so a change under `Eigen/src/Core/arch/AVX512` gets no +AVX-512 compilation at all unless someone applies `all-tests`. Under the `affected-tests` label the tier adds the +configuration that targets the backend the diff touches, through `rules:changes:`: + +| Backend directory | Added configuration | +|---|---| +| `arch/SSE` | x86-64 gcc-10 baseline, AVX, and AVX-512DQ | +| `arch/AVX` | x86-64 gcc-10 AVX and AVX-512DQ | +| `arch/AVX512` | x86-64 gcc-10 AVX-512DQ; `*FP16*` files also get the split gcc-13 AVX512-FP16 compile builds | +| `arch/NEON` | 32-bit arm (aarch64 already runs unconditionally) | +| `arch/AltiVec` | ppc64le gcc-14 | +| `arch/LSX` | loongarch64 gcc-14 | +| `arch/RVV10` | riscv64 gcc-15 | +| `arch/SVE`, `arch/SME` | the full SME build, compile-only | +| `arch/GPU`, `test/*.cu`, `test/gpu_common.h`, `unsupported/test/*.cu`, `unsupported/test/GPU/**` | the CUDA build and test jobs | + +A wider x86 configuration compiles the narrower backends' headers, which is why SSE fans out to three builds. SVE and +SME get compile coverage rather than a selection because their per-SVL test jobs already filter to a curated target +subset through `EIGEN_CI_CTEST_REGEX`, which a selection would fight with. + +AVX512-FP16 headers are guarded by `EIGEN_VECTORIZE_AVX512FP16`, so an AVX512DQ build does not parse them. Changes to +files matching `arch/AVX512/*FP16*` therefore also trigger the existing gcc-13 AVX512-FP16 official and unsupported +builds. Those jobs are compile-only because no current runner can execute AVX512-FP16 instructions. + +The GPU row is the one entry that adds jobs outside the tier rather than an affected build and test pair, because no +affected-tier configuration enables CUDA, HIP or SYCL. In a host-only build there is no `gpu_basic`, `tensor_gpu`, +`cusolver_*` or `cudss_*` target at all, so a diff confined to the GPU test sources selects names that every affected +build reports as unconfigured and hands the test jobs a `-R` regex matching nothing: every step exits 0 and the tier +reads as green having compiled and run nothing. Those paths therefore add the existing CUDA jobs, through the +`affected-tests` entry in `.rules:libeigen:gpu`. They ignore the selection — `EIGEN_CI_BUILD_TARGET` is +`buildtests_gpu` and the test jobs filter on the `gpu` CTest label — so this is coverage of the whole GPU suite, not +of the affected subset. + +`arch/ZVector`, `arch/MSA`, `arch/HVX` and the `arch/HIP` and `arch/SYCL` backends have no matching test +configuration, so a change there gets only the two unconditional jobs and the same hollow result; `gpu-tests` is no +help either, since the GPU jobs it gates are all CUDA. When adding a runner for one of these, add the trigger here +too. + ## Worktree-Safe Formatting Inspect `git status --short` before formatting and preserve unrelated changes. Eigen requires `clang-format-17` diff --git a/.agents/testing.md b/.agents/testing.md index c6e03d05c..f8ed266f8 100644 --- a/.agents/testing.md +++ b/.agents/testing.md @@ -72,7 +72,9 @@ Keep `test/main.h` limited to framework configuration, registration, shared-help Put reusable utilities in a narrowly named helper header; include it from `main.h` only when most tests need it. For compile-failure coverage, use the established `failtest/` pattern. Its `_ok` target must compile and its `_ko` -target must fail with `EIGEN_SHOULD_FAIL_TO_BUILD` defined. +target must fail with `EIGEN_SHOULD_FAIL_TO_BUILD` defined. `_ko` is a `WILL_FAIL` test whose action is a build, so it +cannot tell the intended compile error from any other build failure: keep the construct narrow, and leave the +`RESOURCE_LOCK` that `ei_add_failtest` uses to serialize the suite in place. ## Split Tests diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1a23c957..9142418c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,7 @@ workflow: stages: - checkformat + - select - build - test - benchmark @@ -53,11 +54,16 @@ variables: # Valid CTest labels: Official, Unsupported, gpu, smoketest. EIGEN_CI_CTEST_LABEL: "" EIGEN_CI_CTEST_ARGS: "" + # Affected-test tier: paths to the selection written by the select:tests job. + # Unset for every other job, which keeps their existing behaviour. + EIGEN_CI_BUILD_TARGET_FILE: "" + EIGEN_CI_CTEST_REGEX_FILE: "" include: - "/ci/checkformat.gitlab-ci.yml" - "/ci/common.gitlab-ci.yml" - "/ci/images.gitlab-ci.yml" + - "/ci/select.gitlab-ci.yml" - "/ci/build.linux.gitlab-ci.yml" - "/ci/build.windows.gitlab-ci.yml" - "/ci/test.linux.gitlab-ci.yml" diff --git a/ci/build.linux.gitlab-ci.yml b/ci/build.linux.gitlab-ci.yml index a1108eb79..ae520f340 100644 --- a/ci/build.linux.gitlab-ci.yml +++ b/ci/build.linux.gitlab-ci.yml @@ -653,3 +653,78 @@ build:linux:aarch64:clang-14:default:smoketest: - if: $CI_PIPELINE_SOURCE == "merge_request_event" tags: - saas-linux-large-amd64 + +######## MR Affected Tests ##################################################### +# Opt-in via the `affected-tests` label: build every test the merge request diff +# can reach. The target list comes from the select:tests job; see +# scripts/affected_tests.py. +# +# Two jobs always run under the label and give generic coverage. The rest are +# gated on the backend they compile, so a backend change also builds on the ISA +# it targets -- coverage the default smoke matrix has no job for at all, since +# every smoke build is at baseline ISA. The trigger rules are shared with the +# paired test jobs; see .rules:libeigen:affected-tests:* in +# ci/common.gitlab-ci.yml. + +.affected:build: + needs: [ select:tests ] + variables: + EIGEN_CI_BUILD_TARGET_FILE: affected/targets.txt + +##### Always, under the label ################################################## + +build:linux:cross:x86-64:gcc-10:avx2:affected: + extends: [ build:linux:cross:x86-64:gcc-10:avx2, .affected:build ] + rules: !reference [.rules:libeigen:affected-tests, rules] + +build:linux:cross:aarch64:gcc-10:default:affected: + extends: [ build:linux:cross:aarch64:gcc-10:default, .affected:build ] + rules: !reference [.rules:libeigen:affected-tests, rules] + +##### Backend-triggered ######################################################## + +build:linux:cross:x86-64:gcc-10:default:affected: + extends: [ build:linux:cross:x86-64:gcc-10:default, .affected:build ] + rules: !reference [.rules:libeigen:affected-tests:sse, rules] + +build:linux:cross:x86-64:gcc-10:avx:affected: + extends: [ build:linux:cross:x86-64:gcc-10:avx, .affected:build ] + rules: !reference [.rules:libeigen:affected-tests:avx, rules] + +build:linux:cross:x86-64:gcc-10:avx512dq:affected: + extends: [ build:linux:cross:x86-64:gcc-10:avx512dq, .affected:build ] + rules: !reference [.rules:libeigen:affected-tests:avx512, rules] + +# The FP16 builds reuse the existing split compile-only jobs; there is no runner +# with AVX512-FP16 hardware for a paired test job. +build:linux:cross:x86-64:gcc-13:avx512fp16:official:affected: + extends: build:linux:cross:x86-64:gcc-13:avx512fp16:official + rules: !reference [.rules:libeigen:affected-tests:avx512fp16, rules] + +build:linux:cross:x86-64:gcc-13:avx512fp16:unsupported:affected: + extends: build:linux:cross:x86-64:gcc-13:avx512fp16:unsupported + rules: !reference [.rules:libeigen:affected-tests:avx512fp16, rules] + +# 32-bit arm: a distinct NEON code path from the aarch64 job above. +build:linux:cross:arm:gcc-10:default:affected: + extends: [ build:linux:cross:arm:gcc-10:default, .affected:build ] + rules: !reference [.rules:libeigen:affected-tests:neon, rules] + +build:linux:cross:ppc64le:gcc-14:default:affected: + extends: [ build:linux:cross:ppc64le:gcc-14:default, .affected:build ] + rules: !reference [.rules:libeigen:affected-tests:altivec, rules] + +build:linux:cross:loongarch64:gcc-14:default:affected: + extends: [ build:linux:cross:loongarch64:gcc-14:default, .affected:build ] + rules: !reference [.rules:libeigen:affected-tests:lsx, rules] + +build:linux:riscv64:gcc-15:default:affected: + extends: [ build:linux:riscv64:gcc-15:default, .affected:build ] + rules: !reference [.rules:libeigen:affected-tests:rvv10, rules] + +# SVE and SME get compile coverage of the whole suite under the streaming-mode +# flags rather than a selection: the per-SVL test jobs run a curated target +# subset through EIGEN_CI_CTEST_REGEX, which a selection would conflict with. +build:linux:cross:sme:gcc-14:full:affected: + extends: build:linux:cross:sme:gcc-14:full + rules: !reference [.rules:libeigen:affected-tests:sve-sme, rules] diff --git a/ci/common.gitlab-ci.yml b/ci/common.gitlab-ci.yml index e46efae46..fd095a967 100644 --- a/ci/common.gitlab-ci.yml +++ b/ci/common.gitlab-ci.yml @@ -19,6 +19,91 @@ - if: $CI_PIPELINE_SOURCE == "web" && $CI_PROJECT_NAMESPACE == "libeigen" - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/all-tests/" - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/gpu-tests/" + # No affected-tier configuration enables CUDA, HIP or SYCL, so the GPU test + # targets exist in none of them: a diff confined to these paths selects + # targets every affected build reports as unconfigured and a -R regex that + # matches nothing, and the tier goes green having compiled and run nothing. + # These jobs close that gap the way ppc64le closes it for AltiVec. They + # ignore the selection and build buildtests_gpu, which covers every GPU + # test rather than the affected subset. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/GPU/**/* + - test/*.cu + - test/gpu_common.h + - unsupported/test/*.cu + - unsupported/test/GPU/**/* + +.rules:libeigen:affected-tests: + # Opt-in middle tier between the fixed smoke list and the full `all-tests` + # matrix: every test the diff can reach, on a small set of platforms. + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + +# The rest of the affected tier is gated on the backend a job compiles as well +# as on the label. A build job and its paired test job share one rule set: a +# test job whose build did not run has nothing to execute. A wider x86 +# configuration compiles the narrower backends' headers, so an SSE change +# triggers the AVX and AVX-512 rules too. +.rules:libeigen:affected-tests:sse: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/SSE/**/* + +.rules:libeigen:affected-tests:avx: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/SSE/**/* + - Eigen/src/Core/arch/AVX/**/* + +.rules:libeigen:affected-tests:avx512: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/SSE/**/* + - Eigen/src/Core/arch/AVX/**/* + - Eigen/src/Core/arch/AVX512/**/* + +# The AVX512-FP16 headers are excluded unless -mavx512fp16 is active, so the +# AVX512DQ build does not parse them. +.rules:libeigen:affected-tests:avx512fp16: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/AVX512/*FP16* + +.rules:libeigen:affected-tests:neon: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/NEON/**/* + +.rules:libeigen:affected-tests:altivec: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/AltiVec/**/* + +.rules:libeigen:affected-tests:lsx: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/LSX/**/* + +.rules:libeigen:affected-tests:rvv10: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/RVV10/**/* + +.rules:libeigen:affected-tests:sve-sme: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "libeigen" && $CI_MERGE_REQUEST_LABELS =~ "/affected-tests/" + changes: + - Eigen/src/Core/arch/SVE/**/* + - Eigen/src/Core/arch/SME/**/* .rules:libeigen:scheduled-or-web: rules: diff --git a/ci/scripts/build.linux.script.sh b/ci/scripts/build.linux.script.sh index af324b388..c8cd496d3 100755 --- a/ci/scripts/build.linux.script.sh +++ b/ci/scripts/build.linux.script.sh @@ -25,6 +25,65 @@ cmake -G Ninja \ ${launchers} \ ${EIGEN_CI_ADDITIONAL_ARGS} ${rootdir} +# The affected-tests tier (see scripts/affected_tests.py) passes its selection +# as a file rather than a variable so the list is not bounded by CI variable +# limits. The file holds "NONE" or one target per line; the full-suite form is +# "buildtests" plus the targets it does not aggregate, and so takes the same +# path as any other list. +# Targets that this configuration did not register (optional dependencies such +# as CHOLMOD, CUDA or SYCL) are dropped here: ninja aborts on an unknown target, +# and this is the first point that knows what CMake actually configured. +selected_targets="" +if [[ -n "${EIGEN_CI_BUILD_TARGET_FILE}" ]]; then + target_file="${EIGEN_CI_BUILD_TARGET_FILE}" + [[ "${target_file}" = /* ]] || target_file="${rootdir}/${target_file}" + # Fail loudly rather than falling through to the default target: a missing + # selection would otherwise silently build the entire test suite. + if [[ ! -f "${target_file}" ]]; then + echo "EIGEN_CI_BUILD_TARGET_FILE=${EIGEN_CI_BUILD_TARGET_FILE} does not exist." >&2 + echo "The select:tests artifact is missing; refusing to guess a build target." >&2 + exit 1 + fi + requested=$(cat "${target_file}") + if [[ "${requested}" == "NONE" ]]; then + echo "No tests are affected by this merge request; nothing to build." + cd ${rootdir} + set +x + return 0 2>/dev/null || exit 0 + else + { set +x; } 2>/dev/null + # The runner sources this script under `set -eo pipefail`, so every command + # substitution below has to end in a success status: a failed ninja, or a + # grep that legitimately counts zero lines, would otherwise abandon the job + # before the checks that are meant to report it. + configured=$(ninja -t targets all 2>/dev/null | sed -n 's/^\([A-Za-z_0-9]*\): phony$/\1/p' | sort -u || true) + # An empty query means ninja is unusable, not that nothing is configured. + # Without this the intersection below would be empty and the job would + # trivially "succeed" having built nothing. + if [[ -z "${configured}" ]]; then + echo "Could not enumerate configured targets via 'ninja -t targets'." >&2 + exit 1 + fi + requested_targets=$(echo "${requested}" | sort -u) + selected_targets=$(comm -12 <(echo "${requested_targets}") <(echo "${configured}")) + unconfigured=$(comm -23 <(echo "${requested_targets}") <(echo "${configured}")) + nrequested=$(echo "${requested_targets}" | grep -c . || true) + nselected=$(echo "${selected_targets}" | grep -c . || true) + echo "Affected selection: ${nselected} of ${nrequested} requested targets are configured here." + if [[ -n "${unconfigured}" ]]; then + echo "Not configured in this build: $(echo "${unconfigured}" | tr '\n' ' ')" + fi + set -x + if [[ -z "${selected_targets}" ]]; then + echo "None of the affected tests exist in this configuration; nothing to build." + cd ${rootdir} + set +x + return 0 2>/dev/null || exit 0 + fi + EIGEN_CI_BUILD_TARGET=$(echo "${selected_targets}" | tr '\n' ' ') + fi +fi + target="" if [[ ${EIGEN_CI_BUILD_TARGET} ]]; then target="--target ${EIGEN_CI_BUILD_TARGET}" @@ -69,13 +128,67 @@ default_batch=$((njobs * 8)) default_batch=$((default_batch > 96 ? default_batch : 96)) batch_size=${EIGEN_CI_BUILD_BATCH_SIZE:-${default_batch}} shuffled=false -# The batch path resolves a target's dependency graph via `ninja -t query`, -# which takes a single target name. A multi-target EIGEN_CI_BUILD_TARGET (a -# space-separated list, e.g. the SME cross-build's product_* targets) would be -# passed as one bogus name and make the query fail, aborting the job before any -# build runs. Skip batching for a list and let the plain `cmake --build -# --target t1 t2 ...` below build it directly (it handles multiple targets). -if [[ -n "${EIGEN_CI_BUILD_TARGET}" && "${EIGEN_CI_BUILD_TARGET}" != *[[:space:]]* ]] && command -v ninja >/dev/null 2>&1; then +# An affected-tests selection names CMake targets, and most of those are +# aggregates: "buildtests", and the parent of every split test (bdcsvd depends +# on bdcsvd_1..bdcsvd_41). The batch loop can only spread apart the targets it +# is handed, so an unexpanded parent puts a whole test family in one batch and +# lets all of its parts co-run. expand_to_leaves resolves them first. +# +# CMake emits every aggregate as a phony edge, so descending through phony +# edges alone reaches the compiles and links without walking into object files: +# buildtests -> test/bdcsvd_1, bdcsvd -> test/bdcsvd -> test/bdcsvd_1, and a +# plain executable such as bug1213 -> test/bug1213, which is already a link +# edge and stays as it is. +expand_to_leaves() { + local scratch depth + scratch=$(mktemp -d) + printf '%s\n' ${1} | awk 'NF' | sort -u > "${scratch}/frontier" + : > "${scratch}/leaves" + # Every level costs one ninja invocation over the whole frontier (about 0.1s + # for the full suite). Three levels are enough for the graph above; the cap + # only bounds an unexpected cycle. + for depth in 1 2 3 4 5 6 7 8; do + [[ -s "${scratch}/frontier" ]] || break + # A query that cannot be answered is not a reason to drop targets: leave + # the frontier unexpanded and let ninja resolve it during the build. + ninja -t query $(cat "${scratch}/frontier") > "${scratch}/query" 2>/dev/null || break + awk ' + function flush() { + if (name == "") return + if (rule == "phony" && ninputs > 0) { + for (i = 1; i <= ninputs; i++) print "N", inputs[i] + } else { + print "L", name + } + name = ""; rule = ""; ninputs = 0; section = "" + } + /^[^ ]/ { flush(); name = $0; sub(/:$/, "", name); next } + /^ input:/ { rule = $2; section = "input"; next } + /^ outputs:/ { section = ""; next } + section == "input" && /^ / { inputs[++ninputs] = $1 } + END { flush() } + ' "${scratch}/query" > "${scratch}/classified" + sed -n 's/^L //p' "${scratch}/classified" >> "${scratch}/leaves" + sed -n 's/^N //p' "${scratch}/classified" | sort -u > "${scratch}/next" + mv "${scratch}/next" "${scratch}/frontier" + done + # Whatever is still unexpanded -- a failed query or a chain past the cap -- + # is built as named. Empty after a normal walk. + cat "${scratch}/frontier" >> "${scratch}/leaves" + sort -u "${scratch}/leaves" + rm -rf "${scratch}" +} + +deps="" +if [[ -n "${selected_targets}" ]] && command -v ninja >/dev/null 2>&1; then + { set +x; } 2>/dev/null + deps=$(expand_to_leaves "${selected_targets}") +# The meta-target path below quotes EIGEN_CI_BUILD_TARGET into `ninja -t query`, +# so a space-separated list (the SME cross-build's product_* targets) would go in +# as one bogus name and make the query fail, aborting the job before any build +# runs. Skip batching for a list and let the plain `cmake --build --target t1 t2 +# ...` at the end build it directly; it handles multiple targets. +elif [[ -n "${EIGEN_CI_BUILD_TARGET}" && "${EIGEN_CI_BUILD_TARGET}" != *[[:space:]]* ]] && command -v ninja >/dev/null 2>&1; then # Suppress xtrace while extracting and shuffling the target list # to avoid dumping ~1200 lines to the CI log. { set +x; } 2>/dev/null @@ -91,6 +204,9 @@ if [[ -n "${EIGEN_CI_BUILD_TARGET}" && "${EIGEN_CI_BUILD_TARGET}" != *[[:space:] deps="$inner" fi fi +fi + +if [[ -n "${deps}" ]]; then # Deterministic shuffle: hash each target name and sort by hash. # Stable across runs (helps ninja's .ninja_log and build caches), # portable (no shuf dependency), and spreads same-family targets apart. diff --git a/ci/scripts/test.linux.script.sh b/ci/scripts/test.linux.script.sh index 5c537d326..36f9ba334 100755 --- a/ci/scripts/test.linux.script.sh +++ b/ci/scripts/test.linux.script.sh @@ -5,6 +5,37 @@ set -x rootdir=`pwd` + +# The affected-tests tier (see scripts/affected_tests.py) passes its CTest +# filter as a file rather than a variable so the regex is not bounded by CI +# variable limits. "ALL" means run everything the paired build produced, +# "NONE" means the merge request affects no test at all. +if [[ -n "${EIGEN_CI_CTEST_REGEX_FILE}" ]]; then + regex_file="${EIGEN_CI_CTEST_REGEX_FILE}" + [[ "${regex_file}" = /* ]] || regex_file="${rootdir}/${regex_file}" + # Fail loudly rather than falling through: a missing selection would + # otherwise silently run the whole suite against a partial build. + if [[ ! -f "${regex_file}" ]]; then + echo "EIGEN_CI_CTEST_REGEX_FILE=${EIGEN_CI_CTEST_REGEX_FILE} does not exist." >&2 + echo "The select:tests artifact is missing; refusing to guess a test filter." >&2 + exit 1 + fi + selection=$(cat "${regex_file}") + case "${selection}" in + NONE) + echo "No tests are affected by this merge request; nothing to run." + set +x + return 0 2>/dev/null || exit 0 + ;; + ALL) + EIGEN_CI_CTEST_REGEX="" + ;; + *) + EIGEN_CI_CTEST_REGEX="${selection}" + ;; + esac +fi + cd ${EIGEN_CI_BUILDDIR} target="" diff --git a/ci/select.gitlab-ci.yml b/ci/select.gitlab-ci.yml new file mode 100644 index 000000000..e62d90808 --- /dev/null +++ b/ci/select.gitlab-ci.yml @@ -0,0 +1,34 @@ +# Affected-test selection for the `affected-tests` merge request label. +# SPDX-FileCopyrightText: The Eigen Authors +# SPDX-License-Identifier: MPL-2.0 +# +# Emits the set of tests reachable from the merge request diff, which the +# affected-tier build and test jobs consume. See scripts/affected_tests.py for +# the selection rules and .agents/ci.md for how the tier fits the CI matrix. + +select:tests: + stage: select + image: ubuntu:24.04 + needs: [] + rules: !reference [.rules:libeigen:affected-tests, rules] + variables: + # The selector diffs against the merge-base, which a shallow clone may not + # contain. + GIT_DEPTH: 0 + before_script: + - apt-get update -y + - apt-get install -y --no-install-recommends python3 git + script: + - python3 scripts/test_affected_tests.py + - python3 scripts/affected_tests.py + --base-sha "${CI_MERGE_REQUEST_DIFF_BASE_SHA}" + --output-dir affected + - echo "Build target selection:" && cat affected/targets.txt + artifacts: + when: always + name: "$CI_JOB_NAME_SLUG-$CI_COMMIT_REF_SLUG" + paths: + - affected/ + expire_in: 2 days + tags: + - saas-linux-medium-amd64 diff --git a/ci/test.linux.gitlab-ci.yml b/ci/test.linux.gitlab-ci.yml index 56d7d054b..7d0033ed1 100644 --- a/ci/test.linux.gitlab-ci.yml +++ b/ci/test.linux.gitlab-ci.yml @@ -790,3 +790,63 @@ test:linux:buildsystem: - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH +##### MR Affected Tests ######################################################## +# Paired with the affected build jobs. No CTest label is set: the regex from +# select:tests is the only filter, so a selection spanning Official and +# Unsupported tests runs in one job. +# +# Each job shares its trigger rule with the build job it needs; see +# .rules:libeigen:affected-tests:* in ci/common.gitlab-ci.yml. + +.affected:test: + variables: + EIGEN_CI_CTEST_REGEX_FILE: affected/ctest_regex.txt + +##### Always, under the label ################################################## + +test:linux:x86-64:gcc-10:avx2:affected: + extends: [ .test:linux:x86-64:gcc-10:avx2, .affected:test ] + needs: [ build:linux:cross:x86-64:gcc-10:avx2:affected, select:tests ] + rules: !reference [.rules:libeigen:affected-tests, rules] + +test:linux:aarch64:gcc-10:default:affected: + extends: [ .test:linux:aarch64:gcc-10:default, .affected:test ] + needs: [ build:linux:cross:aarch64:gcc-10:default:affected, select:tests ] + rules: !reference [.rules:libeigen:affected-tests, rules] + +##### Backend-triggered ######################################################## + +test:linux:x86-64:gcc-10:default:affected: + extends: [ .test:linux:x86-64:gcc-10:default, .affected:test ] + needs: [ build:linux:cross:x86-64:gcc-10:default:affected, select:tests ] + rules: !reference [.rules:libeigen:affected-tests:sse, rules] + +test:linux:x86-64:gcc-10:avx:affected: + extends: [ .test:linux:x86-64:gcc-10:avx, .affected:test ] + needs: [ build:linux:cross:x86-64:gcc-10:avx:affected, select:tests ] + rules: !reference [.rules:libeigen:affected-tests:avx, rules] + +test:linux:x86-64:gcc-10:avx512dq:affected: + extends: [ .test:linux:x86-64:gcc-10:avx512dq, .affected:test ] + needs: [ build:linux:cross:x86-64:gcc-10:avx512dq:affected, select:tests ] + rules: !reference [.rules:libeigen:affected-tests:avx512, rules] + +test:linux:arm:gcc-10:default:affected: + extends: [ .test:linux:arm:gcc-10:default, .affected:test ] + needs: [ build:linux:cross:arm:gcc-10:default:affected, select:tests ] + rules: !reference [.rules:libeigen:affected-tests:neon, rules] + +test:linux:ppc64le:gcc-14:default:affected: + extends: [ .test:linux:ppc64le:gcc-14:default, .affected:test ] + needs: [ build:linux:cross:ppc64le:gcc-14:default:affected, select:tests ] + rules: !reference [.rules:libeigen:affected-tests:altivec, rules] + +test:linux:loongarch64:gcc-14:default:affected: + extends: [ .test:linux:loongarch64:gcc-14:default, .affected:test ] + needs: [ build:linux:cross:loongarch64:gcc-14:default:affected, select:tests ] + rules: !reference [.rules:libeigen:affected-tests:lsx, rules] + +test:linux:riscv64:gcc-15:default:affected: + extends: [ .test:linux:riscv64:gcc-15:default, .affected:test ] + needs: [ build:linux:riscv64:gcc-15:default:affected, select:tests ] + rules: !reference [.rules:libeigen:affected-tests:rvv10, rules] diff --git a/cmake/EigenTesting.cmake b/cmake/EigenTesting.cmake index 3ee0ca652..2c7b1cf3a 100644 --- a/cmake/EigenTesting.cmake +++ b/cmake/EigenTesting.cmake @@ -277,6 +277,15 @@ macro(ei_add_failtest testname) # Expect the second test to fail set_tests_properties(${test_target_ko} PROPERTIES WILL_FAIL TRUE) + + # The test action is a build in the shared binary directory, so two failtests + # running at once drive two concurrent builds over one build system. A lock + # shared by the whole suite serializes those while leaving the ordinary tests + # free to run in parallel. It matters most for ${test_target_ko}: WILL_FAIL + # cannot tell the compile error it asserts from a build system that failed for + # an unrelated reason, so a race there passes vacuously. + set_tests_properties(${test_target_ok} ${test_target_ko} PROPERTIES + RESOURCE_LOCK eigen_failtest_build) endmacro() # print a summary of the different options diff --git a/scripts/affected_tests.py b/scripts/affected_tests.py new file mode 100644 index 000000000..b389893b4 --- /dev/null +++ b/scripts/affected_tests.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: The Eigen Authors +# SPDX-License-Identifier: MPL-2.0 + +"""Select the tests affected by a set of changed files. + +Eigen is header-only, so a test is affected by a change exactly when its +translation unit textually includes the changed file. This script builds the +include graph over ``Eigen/``, ``unsupported/Eigen/`` and the test trees, then +maps changed paths to the CMake test targets that reach them. + +The graph follows every ``#include`` regardless of preprocessor guards, so the +closure is a strict superset of the true compile dependency and no test is +dropped because a conditional branch was not taken. Over-approximation is the +safe direction here: the point is to widen coverage relative to the fixed smoke +list, not to minimise work. Changes that invalidate the mapping itself (CMake, +CI, the BLAS/LAPACK shims) fall back to the full ``buildtests`` target. + +Two output files are written, both consumed by ``ci/scripts/build.linux.script.sh`` +and ``ci/scripts/test.linux.script.sh``: + + targets.txt ``NONE``, a newline-separated target list, or the + full-suite list of ``buildtests`` and the targets it does + not aggregate + ctest_regex.txt ``ALL``, ``NONE``, or a CTest ``-R`` regex + +The selected names are CMake target names, not CTest test names: a split test +``foo`` registers ``foo_1``..``foo_N`` as tests but a single ``foo`` target that +aggregates them, so selecting ``foo`` builds and runs every part. Targets that +a given configuration does not register (optional dependencies such as CHOLMOD +or SYCL) are filtered out by the build script, which is the only place that +knows what CMake actually configured. + +Two registrations do not fit that shape: + +* Each compile-failure test under ``failtest/`` compiles its own target from + inside CTest, so those are selected as CTest names and never handed to the + build job. +* ``buildtests`` aggregates the ``ei_add_test`` targets only. A bare + ``add_executable`` such as ``bug1213`` is attached to nothing, so the + full-suite mode has to name those targets next to ``buildtests``. + +A registered target need not compile a ``.cpp``, and need not be named by a +literal: ``ei_add_test`` takes the source extension from +``EIGEN_ADD_TEST_FILENAME_EXTENSION``, which the GPU blocks set to ``cu``, and +the GPU module registers families of tests from ``foreach`` item lists. Both +are read out of the CMake source, so a translation unit under a test root with +no registration the parser can see is an error rather than an assumption. +Those targets are selected like any other; the build script drops them in +configurations that did not register them. ``test/buildsystem/`` is skipped: +its consumer projects are separate CMake projects configured by their own CI +job, so an ``add_executable`` there is not a test registration. +""" + +import argparse +import collections +import fnmatch +import os +import re +import subprocess +import sys + +# Directories scanned to build the include graph. +SCAN_ROOTS = ("Eigen", "unsupported/Eigen", "test", "unsupported/test", "failtest") + +# Directories whose .cpp files are test translation units. +TEST_ROOTS = ("test", "unsupported/test") + +# Subtrees of TEST_ROOTS that are not part of this build. test/buildsystem/ +# holds standalone CMake projects that test:linux:buildsystem configures on +# their own, so their add_executable() calls register targets no configuration +# here has, and their sources are not test translation units. +EXCLUDED_TEST_DIRS = ("test/buildsystem",) + +# Compile-failure suite. ei_add_failtest registers _ok and _ko as +# CTest tests whose test action is a build of an EXCLUDE_FROM_ALL target. +FAILTEST_ROOT = "failtest" + +# Extensions a registered test translation unit can have. ".cu" comes from the +# GPU registrations; see CMAKE_REGISTRATION_RE. +TEST_SOURCE_SUFFIXES = (".cpp", ".cu") + +# Share of the test suite above which an explicit selection is replaced by the +# full ``buildtests`` target. +DEFAULT_MAX_FRACTION = 0.85 + +# Changes matching these patterns cannot affect which tests exist or what they +# cover, so they select nothing. +IGNORED_PATTERNS = ( + ".gitattributes", + ".gitignore", + ".clang-format", + ".clang-tidy", + "*.md", + "*.dox", + "AGENTS.md", + "COPYING*", + "INSTALL", + "README*", + "REUSE.toml", + ".agents/*", + ".gitlab/*", + "LICENSES/*", + "benchmarks/*", + "debug/*", + "demos/*", + "doc/*", + "unsupported/benchmarks/*", + "unsupported/doc/*", +) + +# Changes matching these patterns invalidate the include-graph mapping itself +# (test registration, split counts, the CI drivers, or shim libraries whose +# tests are not modelled here), so they force the full test suite. Checked +# after IGNORED_PATTERNS, so a benchmark's or the docs' own CMakeLists.txt does +# not drag in the whole suite. +FULL_REBUILD_PATTERNS = ( + "CMakeLists.txt", + "*/CMakeLists.txt", + "*.cmake", + "*.cmake.in", + ".gitlab-ci.yml", + "ci/*", + "cmake/*", + "scripts/*", + "blas/*", + "lapack/*", +) + +INCLUDE_RE = re.compile(r'^[ \t]*#[ \t]*include[ \t]*[<"]([^>"]+)[>"]', re.MULTILINE) +# One pass over a test CMakeLists.txt, in source order, because what a +# registration means depends on the state at that point: the source extension +# comes from EIGEN_ADD_TEST_FILENAME_EXTENSION, which the CUDA and HIP blocks +# set to "cu" and unset again, so gpu_basic is test/gpu_basic.cu while its +# neighbours are .cpp; and a name spelled ${var} resolves against the item list +# of the enclosing foreach(), which is how the GPU module registers its +# cusolver_* and cudss_* tests. +CMAKE_REGISTRATION_RE = re.compile( + r"^[ \t]*(?:" + r'(?Pset|unset)\([ \t]*EIGEN_ADD_TEST_FILENAME_EXTENSION[ \t]*"?(?P[A-Za-z_0-9]*)"?' + r"|(?:ei_add_test|ei_add_gpu_test)\([ \t]*" + r"(?P[A-Za-z_][A-Za-z_0-9]*|\$\{[A-Za-z_][A-Za-z_0-9]*\})" + r"|add_executable\([ \t]*(?P[A-Za-z_][A-Za-z_0-9]*)[ \t\r\n]+(?P[^)]*)\)" + r"|foreach\((?P[^)]*)\)" + r"|(?Pendforeach)\(" + r")", + re.MULTILINE, +) +# A bare CMake identifier, used to recognise a foreach loop variable. +CMAKE_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z_0-9]*\Z") +CMAKE_FAILTEST_RE = re.compile( + r'^[ \t]*ei_add_failtest\([ \t]*"?([A-Za-z_][A-Za-z_0-9]*)"?', + re.MULTILINE, +) + + +def _matches(path, patterns): + return any(fnmatch.fnmatch(path, p) for p in patterns) + + +def _under(path, roots): + return any(path.startswith(root + "/") for root in roots) + + +class IncludeGraph: + """Textual ``#include`` graph over the scanned source roots.""" + + def __init__(self, source_dir): + self.source_dir = source_dir + self.files = set() + self._direct = {} + self._by_suffix = {} + self._scan() + + def _scan(self): + for root in SCAN_ROOTS: + abs_root = os.path.join(self.source_dir, root) + if not os.path.isdir(abs_root): + continue + for dirpath, dirnames, filenames in os.walk(abs_root): + dirnames[:] = [d for d in dirnames if not d.startswith(".")] + for name in filenames: + rel = os.path.relpath(os.path.join(dirpath, name), self.source_dir) + self.files.add(rel) + # Index every path suffix so that an include spelled relative to a + # directory outside the scanned roots still resolves. Ambiguous + # suffixes are dropped rather than guessed. + candidates = {} + for rel in self.files: + parts = rel.split("/") + for i in range(len(parts)): + candidates.setdefault("/".join(parts[i:]), []).append(rel) + self._by_suffix = {suffix: matches[0] + for suffix, matches in candidates.items() if len(matches) == 1} + + def read_text(self, rel): + """Contents of a file in the tree, or ``''`` if it cannot be read.""" + try: + with open(os.path.join(self.source_dir, rel), "r", errors="ignore") as handle: + return handle.read() + except OSError: + return "" + + def direct_includes(self, rel): + """Resolved includes of a single file.""" + cached = self._direct.get(rel) + if cached is not None: + return cached + resolved = set() + self._direct[rel] = resolved # placed first: the graph has cycles + directory = os.path.dirname(rel) + for spelling in INCLUDE_RE.findall(self.read_text(rel)): + candidate = os.path.normpath(os.path.join(directory, spelling)) + if candidate in self.files: + resolved.add(candidate) + elif spelling in self.files: + resolved.add(spelling) + elif spelling in self._by_suffix: + resolved.add(self._by_suffix[spelling]) + return resolved + + def closure(self, rel): + """Every file reachable from ``rel`` through includes.""" + seen = set() + stack = [rel] + while stack: + for nxt in self.direct_includes(stack.pop()): + if nxt not in seen: + seen.add(nxt) + stack.append(nxt) + return seen + + +# targets -- test translation unit -> the CMake target that compiles it +# standalone -- targets the ``buildtests`` aggregate does not depend on +# failtests -- failtest translation unit -> the CTest names that compile it +Registrations = collections.namedtuple("Registrations", "targets standalone failtests") + + +def _loop_binding(arguments): + """Names a ``foreach(...)`` binds, or ``None`` when they are not literal.""" + tokens = [token.strip('"') for token in arguments.split()] + if not tokens or not CMAKE_NAME_RE.match(tokens[0]): + return None + variable, items = tokens[0], tokens[1:] + if items[:2] == ["IN", "ITEMS"]: + items = items[2:] + elif items[:1] == ["IN"]: + # IN LISTS and IN ZIP_LISTS iterate variables, not literal names. + return None + return variable, [item for item in items if "$" not in item] + + +def _loop_expand(token, loops): + """Resolve a registration name against the enclosing ``foreach`` bindings.""" + if not token.startswith("$"): + return [token] + name = token[2:-1] + for binding in reversed(loops): + if binding is not None and binding[0] == name: + return binding[1] + return [] + + +def test_registrations(graph): + """Map registered translation units to what CI has to build or run.""" + source_targets = {} + standalone = set() + + def register(source, target): + previous = source_targets.get(source) + if previous is not None and previous != target: + raise ValueError("%s is registered by both %s and %s" % (source, previous, target)) + source_targets[source] = target + + cmake_files = sorted( + rel + for rel in graph.files + if os.path.basename(rel) == "CMakeLists.txt" + and _under(rel, TEST_ROOTS) + and not _under(rel, EXCLUDED_TEST_DIRS) + ) + for cmake_file in cmake_files: + directory = os.path.dirname(cmake_file) + extension = "cpp" + # foreach() bindings in effect, innermost last. A loop over anything + # but a literal item list pushes None so endforeach() stays balanced. + loops = [] + for match in CMAKE_REGISTRATION_RE.finditer(graph.read_text(cmake_file)): + if match.group("scope"): + # unset(), or a set() with no value, restores the default. + extension = match.group("extension") if match.group("scope") == "set" else "" + extension = extension or "cpp" + continue + if match.group("loop") is not None: + loops.append(_loop_binding(match.group("loop"))) + continue + if match.group("endloop"): + if loops: + loops.pop() + continue + if match.group("test"): + for target in _loop_expand(match.group("test"), loops): + source = os.path.normpath( + os.path.join(directory, "%s.%s" % (target, extension))) + if source in graph.files: + register(source, target) + continue + target = match.group("executable") + for token in re.findall(r'"[^"]*"|[^\s]+', match.group("sources")): + token = token.strip('"') + if not token.endswith(TEST_SOURCE_SUFFIXES) or "$" in token: + continue + source = os.path.normpath(os.path.join(directory, token)) + if source in graph.files: + register(source, target) + standalone.add(target) + + failtests = {} + for name in CMAKE_FAILTEST_RE.findall(graph.read_text(FAILTEST_ROOT + "/CMakeLists.txt")): + source = "%s/%s.cpp" % (FAILTEST_ROOT, name) + if source in graph.files: + failtests[source] = (name + "_ok", name + "_ko") + + return Registrations(source_targets, standalone, failtests) + + +def full_suite(graph, reasons): + """Full-suite selection, naming the targets ``buildtests`` does not build.""" + try: + standalone = test_registrations(graph).standalone + except ValueError: + # A broken registration is reported by the paths that depend on the + # mapping; the full suite stays available without it. + standalone = () + return Selection("all", reasons=reasons, standalone=standalone) + + +def reverse_map(graph, sources): + """Map each included file to the test sources that reach it.""" + reverse = {} + for src in sources: + for dep in graph.closure(src): + reverse.setdefault(dep, set()).add(src) + return reverse + + +class Selection: + """Outcome: the full suite, explicit targets, no tests, or an error.""" + + def __init__(self, mode, targets=(), reasons=(), ctest_names=(), standalone=()): + self.mode = mode # "all", "targets", "none", or "error" + self.targets = set(targets) + self.reasons = list(reasons) + # CTest names with no build target of their own. + self.ctest_names = set(ctest_names) + # Targets to name alongside ``buildtests`` in "all" mode. + self.standalone = set(standalone) + + @property + def targets_file(self): + if self.mode == "error": + raise ValueError("an invalid selection has no target file") + if self.mode == "all": + return "".join(name + "\n" for name in ["buildtests"] + sorted(self.standalone)) + if self.mode == "none": + return "NONE\n" + return "".join(name + "\n" for name in sorted(self.targets)) + + @property + def regex_file(self): + if self.mode == "error": + raise ValueError("an invalid selection has no regex file") + if self.mode == "all": + return "ALL\n" + if self.mode == "none": + return "NONE\n" + names = sorted(self.targets) + sorted(self.ctest_names) + return "^(%s)(_[0-9]+)?$\n" % "|".join(re.escape(name) for name in names) + + +def select(graph, changed_files, max_fraction=DEFAULT_MAX_FRACTION): + """Map changed paths to the tests that must run.""" + paths = [] + for path in changed_files: + path = path.strip() + if path and not _matches(path, IGNORED_PATTERNS): + paths.append(path) + if not paths: + return Selection("none", reasons=["no change reaches a test"]) + for path in paths: + if _matches(path, FULL_REBUILD_PATTERNS): + return full_suite(graph, ["%s forces the full suite" % path]) + + try: + registered = test_registrations(graph) + except ValueError as error: + return Selection("error", reasons=[str(error)]) + sources = sorted(registered.targets) + reverse = reverse_map(graph, sources) + failtest_reverse = reverse_map(graph, sorted(registered.failtests)) + + selected = set() + selected_failtests = set() + reasons = [] + for path in paths: + reached_by = set(reverse.get(path, ())) + if path in registered.targets: + reached_by.add(path) + failtests = set(failtest_reverse.get(path, ())) + if path in registered.failtests: + failtests.add(path) + if reached_by or failtests: + selected |= reached_by + selected_failtests |= failtests + continue + if path in graph.files: + # A source file in the tree that nothing includes: either a new + # header not yet wired up or an unregistered translation unit. + roots = TEST_ROOTS + (FAILTEST_ROOT,) + if (_under(path, roots) and not _under(path, EXCLUDED_TEST_DIRS) + and path.endswith(TEST_SOURCE_SUFFIXES)): + return Selection("error", reasons=["%s has no CMake test target" % path]) + reasons.append("%s is in the tree but reaches no test" % path) + continue + # Deleted, renamed, or outside every scanned root: the graph cannot say + # what it affected, so do not guess. + return full_suite(graph, ["%s is not in the include graph" % path]) + + if not selected and not selected_failtests: + return Selection("none", reasons=reasons or ["no change reaches a test"]) + + if len(selected) > max_fraction * len(sources): + reasons.append( + "%d of %d test sources selected (>%.0f%%)" + % (len(selected), len(sources), 100 * max_fraction) + ) + return full_suite(graph, reasons) + + ctest_names = set() + for source in sorted(selected_failtests): + ctest_names.update(registered.failtests[source]) + if ctest_names: + reasons.append("%d compile-failure test(s) build from inside CTest" + % len(selected_failtests)) + return Selection("targets", (registered.targets[s] for s in selected), reasons, + ctest_names=ctest_names) + + +def changed_files_from_git(source_dir, base_sha, head="HEAD"): + """Paths changed between ``base_sha`` and ``head``.""" + result = subprocess.run( + ["git", "diff", "--no-renames", "--name-only", "%s...%s" % (base_sha, head)], + cwd=source_dir, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError("git diff failed: %s" % result.stderr.strip()) + return [line for line in result.stdout.splitlines() if line.strip()] + + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + default_source = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + parser.add_argument("--source-dir", default=default_source, + help="Eigen source tree (default: the tree containing this script)") + parser.add_argument("--base-sha", + help="compute changed files from 'git diff BASE...HEAD'") + parser.add_argument("--head", default="HEAD", help="head revision for --base-sha") + parser.add_argument("--changed-files", + help="read newline-separated changed paths from this file ('-' for stdin)") + parser.add_argument("--output-dir", + help="write targets.txt and ctest_regex.txt here") + parser.add_argument("--max-fraction", type=float, default=DEFAULT_MAX_FRACTION, + help="degrade to the full suite above this fraction (default: %(default)s)") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + + if args.changed_files: + if args.changed_files == "-": + changed = sys.stdin.read().splitlines() + else: + with open(args.changed_files) as handle: + changed = handle.read().splitlines() + elif args.base_sha: + try: + changed = changed_files_from_git(args.source_dir, args.base_sha, args.head) + except RuntimeError as error: + # Without a usable diff there is no basis for narrowing. + print("%s; selecting the full suite" % error, file=sys.stderr) + changed = None + else: + print("one of --base-sha or --changed-files is required", file=sys.stderr) + return 2 + + graph = IncludeGraph(args.source_dir) + if changed is None: + selection = full_suite(graph, ["the merge-base diff is unavailable"]) + else: + selection = select(graph, changed, args.max_fraction) + + print("mode: %s" % selection.mode, file=sys.stderr) + for reason in selection.reasons: + print(" %s" % reason, file=sys.stderr) + if selection.mode == "error": + return 1 + if selection.mode == "targets": + print(" %d targets: %s" % (len(selection.targets), + " ".join(sorted(selection.targets))), file=sys.stderr) + if selection.ctest_names: + print(" %d CTest-only: %s" % (len(selection.ctest_names), + " ".join(sorted(selection.ctest_names))), + file=sys.stderr) + + if args.output_dir: + os.makedirs(args.output_dir, exist_ok=True) + with open(os.path.join(args.output_dir, "targets.txt"), "w") as handle: + handle.write(selection.targets_file) + with open(os.path.join(args.output_dir, "ctest_regex.txt"), "w") as handle: + handle.write(selection.regex_file) + else: + sys.stdout.write(selection.targets_file) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_affected_tests.py b/scripts/test_affected_tests.py new file mode 100644 index 000000000..b2d4a609d --- /dev/null +++ b/scripts/test_affected_tests.py @@ -0,0 +1,554 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: The Eigen Authors +# SPDX-License-Identifier: MPL-2.0 + +"""Unit tests for scripts/affected_tests.py. + +Runs against a synthetic source tree so the expectations do not drift as the +real headers change, plus a few assertions against the checked-out tree that +only depend on properties the selector must always hold. + +Usage: python3 scripts/test_affected_tests.py +""" + +import os +import shutil +import subprocess +import sys +import tempfile + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from affected_tests import ( + IncludeGraph, + Selection, + changed_files_from_git, + full_suite, + select, + test_registrations, +) + +FIXTURE = { + "Eigen/Core": '#include "src/Core/util/Meta.h"\n#include "src/Core/Block.h"\n', + "Eigen/Dense": '#include "Core"\n#include "SVD"\n', + "Eigen/SVD": '#include "Core"\n#include "src/SVD/BDCSVD.h"\n', + "Eigen/src/Core/util/Meta.h": "", + "Eigen/src/Core/Block.h": "", + "Eigen/src/SVD/BDCSVD.h": "", + "Eigen/src/Geometry/Quaternion.h": "", + "test/main.h": "#include \n", + "test/block.cpp": '#include "main.h"\n', + "test/bdcsvd.cpp": '#include "main.h"\n#include \n', + "test/dense.cpp": '#include "main.h"\n#include \n', + "test/multitu.cpp": '#include "main.h"\n', + "test/multitu_main.cpp": '#include "main.h"\n', + "test/gpu_common.h": '#include "main.h"\n', + "test/gpu_basic.cu": '#include "gpu_common.h"\n', + "test/after_gpu.cpp": '#include "main.h"\n', + "test/CMakeLists.txt": """ei_add_test(block) +ei_add_test(bdcsvd) +ei_add_test(dense) +add_executable(multitu multitu.cpp multitu_main.cpp) +set(EIGEN_ADD_TEST_FILENAME_EXTENSION "cu") +ei_add_test(gpu_basic) +unset(EIGEN_ADD_TEST_FILENAME_EXTENSION) +ei_add_test(after_gpu) +""", + # Standalone CMake projects, configured by test:linux:buildsystem alone. + # The two target names differ so that a source shared between them would + # be a duplicate registration if the scan looked at them at all. + "test/buildsystem/consumers/main.cpp": "#include \n", + "test/buildsystem/consumers/installed/CMakeLists.txt": + "add_executable(installed_consumer ../main.cpp)\n", + "test/buildsystem/consumers/subproject/CMakeLists.txt": + "add_executable(subproject_consumer ../main.cpp)\n", + "unsupported/test/extra.cpp": '#include "../../test/main.h"\n', + "unsupported/test/CMakeLists.txt": "ei_add_test(extra)\n", + "failtest/svd_int.cpp": "#include \n", + "failtest/const_block.cpp": "#include \n", + "failtest/CMakeLists.txt": 'ei_add_failtest("svd_int")\nei_add_failtest("const_block")\n', +} + + +def build_fixture(root): + for rel, content in FIXTURE.items(): + path = os.path.join(root, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as handle: + handle.write(content) + + +FAILURES = [] + + +def check(condition, message): + if condition: + return + FAILURES.append(message) + print("FAIL: %s" % message) + + +def targets_of(selection): + return sorted(selection.targets) + + +def test_fixture_graph(root): + graph = IncludeGraph(root) + sources = sorted(test_registrations(graph).targets) + check(sources == ["test/after_gpu.cpp", "test/bdcsvd.cpp", "test/block.cpp", + "test/dense.cpp", "test/gpu_basic.cu", "test/multitu.cpp", + "test/multitu_main.cpp", "unsupported/test/extra.cpp"], + "test sources discovered in both trees, got %s" % sources) + + # A leaf header reaches only the tests whose closure includes it. + sel = select(graph, ["Eigen/src/SVD/BDCSVD.h"]) + check(sel.mode == "targets" and targets_of(sel) == ["bdcsvd", "dense"], + "BDCSVD.h selects bdcsvd and dense, got %s (%s)" % (targets_of(sel), sel.mode)) + + # A hub header reaches everything and degrades to the full suite. + sel = select(graph, ["Eigen/src/Core/util/Meta.h"]) + check(sel.mode == "all", "Meta.h degrades to the full suite, got %s" % sel.mode) + + # ... but not when the threshold allows the explicit list. + sel = select(graph, ["Eigen/src/Core/util/Meta.h"], max_fraction=1.0) + check(sel.mode == "targets" and len(sel.targets) == 7, + "Meta.h reaches all seven targets, got %s" % targets_of(sel)) + + # Umbrella indirection is followed: Dense -> SVD -> BDCSVD.h. + sel = select(graph, ["Eigen/Dense"]) + check(sel.mode == "targets" and targets_of(sel) == ["dense"], + "Eigen/Dense selects only the test including it, got %s" % targets_of(sel)) + + # A header that no test reaches selects nothing. + sel = select(graph, ["Eigen/src/Geometry/Quaternion.h"]) + check(sel.mode == "none", "an unreached header selects nothing, got %s" % sel.mode) + + # A changed test source selects itself. + sel = select(graph, ["test/block.cpp"]) + check(sel.mode == "targets" and targets_of(sel) == ["block"], + "a changed test selects itself, got %s" % targets_of(sel)) + + # Multi-translation-unit executables map every source to the registered + # target rather than assuming each basename is a target. + sel = select(graph, ["test/multitu_main.cpp"]) + check(sel.mode == "targets" and targets_of(sel) == ["multitu"], + "a secondary translation unit selects its executable, got %s" % targets_of(sel)) + + # Documentation, benchmarks and metadata select nothing, including their + # own CMakeLists.txt -- which must not trip the full-rebuild rule. + sel = select(graph, ["doc/TopicLazyEvaluation.dox", "README.md", ".agents/ci.md", + "benchmarks/Core/bench_reductions.cpp", + "unsupported/benchmarks/GPU/CMakeLists.txt", + "doc/CMakeLists.txt", "debug/gdb/printers.py"]) + check(sel.mode == "none", "docs and benchmarks select nothing, got %s (%s)" + % (sel.mode, sel.reasons)) + + # CMake and CI changes invalidate the mapping. + for path in ["CMakeLists.txt", "test/CMakeLists.txt", "cmake/EigenTesting.cmake", + "ci/scripts/build.linux.script.sh", ".gitlab-ci.yml", "blas/level3_impl.h"]: + sel = select(graph, [path]) + check(sel.mode == "all", "%s forces the full suite, got %s" % (path, sel.mode)) + + # An unknown path (deleted or renamed away) is not guessed at. + sel = select(graph, ["Eigen/src/Core/util/Removed.h"]) + check(sel.mode == "all", "an unknown path forces the full suite, got %s" % sel.mode) + + # A new test source must not disappear as an unconfigured target if its + # CMake registration was forgotten. + new_test = os.path.join(root, "test", "brand_new.cpp") + with open(new_test, "w") as handle: + handle.write('#include "main.h"\n') + graph = IncludeGraph(root) + sel = select(graph, ["test/brand_new.cpp"]) + check(sel.mode == "error", "an unregistered test source fails selection, got %s" % sel.mode) + + # A full-suite change takes precedence regardless of path order; this is + # the normal path when a new source and its CMake registration land together. + sel = select(graph, ["test/brand_new.cpp", "test/CMakeLists.txt"]) + check(sel.mode == "all", "CMake changes force the full suite before source validation") + + with open(os.path.join(root, "test", "CMakeLists.txt"), "a") as handle: + handle.write("ei_add_test(brand_new)\n") + graph = IncludeGraph(root) + sel = select(graph, ["test/brand_new.cpp"]) + check(sel.mode == "targets" and "brand_new" in sel.targets, + "a registered new test source is selected, got %s" % targets_of(sel)) + os.remove(new_test) + + # Mixed changes union their selections. + graph = IncludeGraph(root) + sel = select(graph, ["Eigen/src/SVD/BDCSVD.h", "unsupported/test/extra.cpp"]) + check(sel.mode == "targets" and targets_of(sel) == ["bdcsvd", "dense", "extra"], + "mixed changes union, got %s" % targets_of(sel)) + + +def test_buildsystem_fixtures(root): + """test/buildsystem/ registers nothing: it is not part of this build.""" + graph = IncludeGraph(root) + registered = test_registrations(graph) + check("test/buildsystem/consumers/main.cpp" not in registered.targets, + "a buildsystem consumer is not a test translation unit") + check("installed_consumer" not in registered.standalone + and "subproject_consumer" not in registered.standalone, + "a buildsystem consumer is not a standalone target, got %s" + % sorted(registered.standalone)) + check("consumer" not in full_suite(graph, []).targets_file, + "full mode does not name a target no configuration has, got %r" + % full_suite(graph, []).targets_file) + + # Falling to "reaches no test" is the honest answer: test:linux:buildsystem + # covers these on every merge request through its own changes: rule. + sel = select(graph, ["test/buildsystem/consumers/main.cpp"]) + check(sel.mode == "none", + "a buildsystem source reaches no test, got %s (%s)" % (sel.mode, sel.reasons)) + + # A .cpp there is not an unregistered test source either, so adding one + # must not fail the selection. + extra = os.path.join(root, "test", "buildsystem", "consumers", "extra.cpp") + with open(extra, "w") as handle: + handle.write("#include \n") + try: + sel = select(IncludeGraph(root), ["test/buildsystem/consumers/extra.cpp"]) + check(sel.mode == "none", + "a new buildsystem source is not an unregistered test, got %s" % sel.mode) + finally: + os.remove(extra) + + +def test_failtests(root): + """The compile-failure suite is selected as CTest names, never as targets.""" + graph = IncludeGraph(root) + registered = test_registrations(graph) + check(sorted(registered.failtests) == ["failtest/const_block.cpp", "failtest/svd_int.cpp"], + "failtests are discovered, got %s" % sorted(registered.failtests)) + check(registered.failtests["failtest/svd_int.cpp"] == ("svd_int_ok", "svd_int_ko"), + "each failtest registers an _ok and a _ko CTest test") + check(not any(t.startswith("svd_int") for t in registered.targets.values()), + "failtests are not build targets") + + # A changed failtest source runs itself and nothing else. It has no build + # target: CTest compiles it as the test action. + sel = select(graph, ["failtest/svd_int.cpp"]) + check(sel.mode == "targets" and targets_of(sel) == [], + "a failtest selects no build target, got %s" % targets_of(sel)) + check(sorted(sel.ctest_names) == ["svd_int_ko", "svd_int_ok"], + "a failtest selects its CTest names, got %s" % sorted(sel.ctest_names)) + check(sel.targets_file == "", "a failtest-only selection builds nothing, got %r" + % sel.targets_file) + check(sel.regex_file == "^(svd_int_ko|svd_int_ok)(_[0-9]+)?$\n", + "a failtest-only regex names both parts, got %r" % sel.regex_file) + + # A header reaches its failtests through the same include closure as its + # tests, so both are selected together. + sel = select(graph, ["Eigen/src/SVD/BDCSVD.h"]) + check(sel.mode == "targets" and targets_of(sel) == ["bdcsvd", "dense"], + "BDCSVD.h still selects its tests, got %s" % targets_of(sel)) + check(sorted(sel.ctest_names) == ["svd_int_ko", "svd_int_ok"], + "BDCSVD.h also selects the failtest reaching it, got %s" % sorted(sel.ctest_names)) + check(sel.regex_file == "^(bdcsvd|dense|svd_int_ko|svd_int_ok)(_[0-9]+)?$\n", + "targets and CTest-only names share one regex, got %r" % sel.regex_file) + + # A failtest with no ei_add_failtest call is an error, like a test source + # with no registration. + orphan = os.path.join(root, "failtest", "orphan.cpp") + with open(orphan, "w") as handle: + handle.write("#include \n") + try: + sel = select(IncludeGraph(root), ["failtest/orphan.cpp"]) + check(sel.mode == "error", + "an unregistered failtest fails selection, got %s" % sel.mode) + finally: + os.remove(orphan) + + +def test_cuda_registrations(root): + """ei_add_test compiles .cu while EIGEN_ADD_TEST_FILENAME_EXTENSION is set.""" + graph = IncludeGraph(root) + registered = test_registrations(graph) + check(registered.targets.get("test/gpu_basic.cu") == "gpu_basic", + "a .cu test registers its own source, got %s" + % registered.targets.get("test/gpu_basic.cu")) + check("test/gpu_basic.cpp" not in registered.targets, + "the .cu registration does not synthesise a .cpp source") + check(registered.targets.get("test/after_gpu.cpp") == "after_gpu", + "unset restores the default extension, got %s" + % registered.targets.get("test/after_gpu.cpp")) + + sel = select(graph, ["test/gpu_basic.cu"]) + check(sel.mode == "targets" and targets_of(sel) == ["gpu_basic"], + "a changed .cu source selects its target, got %s (%s)" + % (targets_of(sel), sel.mode)) + + # A header only the .cu test includes must still reach it: a configuration + # without CUDA reports the target as unconfigured, which is not the same as + # reporting that no test is affected. + sel = select(graph, ["test/gpu_common.h"]) + check(sel.mode == "targets" and targets_of(sel) == ["gpu_basic"], + "a GPU-only header selects the .cu test, got %s (%s)" + % (targets_of(sel), sel.mode)) + + # An unregistered .cu is an error, like an unregistered .cpp. + orphan = os.path.join(root, "test", "gpu_orphan.cu") + with open(orphan, "w") as handle: + handle.write('#include "main.h"\n') + try: + sel = select(IncludeGraph(root), ["test/gpu_orphan.cu"]) + check(sel.mode == "error", "an unregistered .cu fails selection, got %s" % sel.mode) + finally: + os.remove(orphan) + + +FOREACH_FIXTURE = { + "test/main.h": "", + "unsupported/test/GPU/gpu_test_helpers.h": '#include "../../../test/main.h"\n', + "unsupported/test/GPU/device_matrix.cpp": '#include "gpu_test_helpers.h"\n', + "unsupported/test/GPU/cusolver_llt.cpp": '#include "gpu_test_helpers.h"\n', + "unsupported/test/GPU/cusolver_qr.cpp": '#include "gpu_test_helpers.h"\n', + "unsupported/test/GPU/from_variable.cpp": '#include "gpu_test_helpers.h"\n', + "unsupported/test/GPU/unregistered.cpp": '#include "gpu_test_helpers.h"\n', + "unsupported/test/GPU/CMakeLists.txt": """function(ei_add_gpu_test test_name) + ei_add_test(${test_name} "" "CUDA::cudart_static") + foreach(t ${_targets}) + add_dependencies(buildtests_gpu ${t}) + endforeach() +endfunction() + +ei_add_gpu_test(device_matrix EXTRA_LIBS CUDA::cublas) + +foreach(_cusolver_test IN ITEMS cusolver_llt cusolver_qr) + ei_add_gpu_test(${_cusolver_test} EXTRA_LIBS CUDA::cusolver) +endforeach() + +foreach(_computed IN LISTS SOME_LIST) + ei_add_gpu_test(${_computed}) +endforeach() +""", +} + + +def test_foreach_registrations(): + """GPU registrations are derived from the calls, including foreach items.""" + root = tempfile.mkdtemp(prefix="eigen-affected-foreach-") + try: + for rel, content in FOREACH_FIXTURE.items(): + path = os.path.join(root, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as handle: + handle.write(content) + graph = IncludeGraph(root) + sources = test_registrations(graph).targets + + check(sources.get("unsupported/test/GPU/device_matrix.cpp") == "device_matrix", + "a literal ei_add_gpu_test registers its source, got %s" + % sources.get("unsupported/test/GPU/device_matrix.cpp")) + check(sources.get("unsupported/test/GPU/cusolver_llt.cpp") == "cusolver_llt" + and sources.get("unsupported/test/GPU/cusolver_qr.cpp") == "cusolver_qr", + "foreach(... IN ITEMS ...) expands to one registration per item, got %s" + % sorted(sources)) + + # ei_add_test(${test_name}) inside the wrapper's own body is a function + # parameter, not a loop item, so it must not register anything. + check("unsupported/test/GPU/test_name.cpp" not in sources + and "test_name" not in set(sources.values()), + "the wrapper's own parameter is not a registration, got %s" % sorted(sources)) + + # A .cpp whose only registration iterates a variable, and one with no + # registration at all, must both reach the error path rather than being + # assumed registered because of where they live. + for path in ("unsupported/test/GPU/from_variable.cpp", + "unsupported/test/GPU/unregistered.cpp"): + check(path not in sources, "%s is not registered, got %s" % (path, sources.get(path))) + sel = select(graph, [path]) + check(sel.mode == "error", + "%s fails selection, got %s (%s)" % (path, sel.mode, sel.reasons)) + + # The header the registered tests share still reaches them. + sel = select(graph, ["unsupported/test/GPU/gpu_test_helpers.h"], max_fraction=1.0) + check(sel.mode == "targets" + and targets_of(sel) == ["cusolver_llt", "cusolver_qr", "device_matrix"], + "the GPU header reaches its registered tests, got %s (%s)" + % (targets_of(sel), sel.mode)) + finally: + shutil.rmtree(root, ignore_errors=True) + + +def test_output_encoding(): + sel = Selection("targets", ["adjoint", "bdcsvd"]) + check(sel.targets_file == "adjoint\nbdcsvd\n", + "target list is newline separated, got %r" % sel.targets_file) + check(sel.regex_file == "^(adjoint|bdcsvd)(_[0-9]+)?$\n", + "regex matches every part, got %r" % sel.regex_file) + + sel = Selection("targets", ["adjoint"], ctest_names=["x_ok", "x_ko"]) + check(sel.targets_file == "adjoint\n", + "CTest-only names stay out of the target list, got %r" % sel.targets_file) + check(sel.regex_file == "^(adjoint|x_ko|x_ok)(_[0-9]+)?$\n", + "CTest-only names join the regex, got %r" % sel.regex_file) + + check(Selection("all").targets_file == "buildtests\n", "full mode builds everything") + # buildtests does not aggregate a bare add_executable, so full mode has to + # name those targets or they stop being compiled. + check(Selection("all", standalone=["bug1213"]).targets_file == "buildtests\nbug1213\n", + "full mode names the targets buildtests omits, got %r" + % Selection("all", standalone=["bug1213"]).targets_file) + check(Selection("all").regex_file == "ALL\n", "full mode runs everything") + check(Selection("none").targets_file == "NONE\n", "empty mode builds nothing") + check(Selection("none").regex_file == "NONE\n", "empty mode runs nothing") + + +def test_git_rename_paths(): + root = tempfile.mkdtemp(prefix="eigen-affected-git-") + try: + old_path = os.path.join(root, "Eigen", "src", "Old.h") + new_path = os.path.join(root, "Eigen", "src", "New.h") + os.makedirs(os.path.dirname(old_path)) + with open(old_path, "w") as handle: + handle.write("// test\n") + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + subprocess.run(["git", "add", "Eigen/src/Old.h"], cwd=root, check=True) + subprocess.run( + ["git", "-c", "user.name=Eigen Tests", "-c", "user.email=eigen@example.com", + "commit", "-qm", "base"], + cwd=root, + check=True, + ) + base = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=root, check=True, capture_output=True, text=True + ).stdout.strip() + os.rename(old_path, new_path) + subprocess.run( + ["git", "add", "Eigen/src/Old.h", "Eigen/src/New.h"], cwd=root, check=True + ) + subprocess.run( + ["git", "-c", "user.name=Eigen Tests", "-c", "user.email=eigen@example.com", + "commit", "-qm", "rename"], + cwd=root, + check=True, + ) + changed = changed_files_from_git(root, base) + check(changed == ["Eigen/src/New.h", "Eigen/src/Old.h"], + "renames expose both paths, got %s" % changed) + finally: + shutil.rmtree(root, ignore_errors=True) + + +def test_real_tree(): + """Properties that must hold against the checked-out tree.""" + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + if not os.path.isdir(os.path.join(root, "Eigen", "src")): + print("skipping real-tree checks: not an Eigen source tree") + return + graph = IncludeGraph(root) + registered = test_registrations(graph) + source_targets = registered.targets + check(len(source_targets) > 200, + "real tree has many test sources, got %d" % len(source_targets)) + + # bug1213 and ulp_accuracy are manual add_executable targets: nothing + # aggregates them, so the full-suite selection has to name them or those + # regressions stop being compiled. Asserted as an exact set, because a + # name that reaches this set without belonging in it makes the build jobs' + # "not configured in this build" diagnostic permanently non-empty. + check(registered.standalone == {"bug1213", "ulp_accuracy"}, + "unexpected standalone target set, got %s" % sorted(registered.standalone)) + if "test/bug1213.cpp" in graph.files: + check("\nbug1213\n" in full_suite(graph, []).targets_file, + "full mode names bug1213, got %r" % full_suite(graph, []).targets_file) + + # test/buildsystem/ is under a test root but is not part of this build. + check(not any(rel.startswith("test/buildsystem/") for rel in source_targets), + "no buildsystem fixture is registered, got %s" + % sorted(rel for rel in source_targets if rel.startswith("test/buildsystem/"))) + + # The GPU tests are registered as .cu through EIGEN_ADD_TEST_FILENAME_EXTENSION. + # Configurations without CUDA report them as unconfigured; dropping them from + # the mapping instead would report that no test is affected at all. + if "test/gpu_basic.cu" in graph.files: + check(source_targets.get("test/gpu_basic.cu") == "gpu_basic", + "test/gpu_basic.cu maps to gpu_basic, got %s" + % source_targets.get("test/gpu_basic.cu")) + sel = select(graph, ["test/gpu_common.h"]) + check(sel.mode == "targets" and "gpu_basic" in sel.targets, + "test/gpu_common.h reaches gpu_basic, got %s (%s)" + % (sorted(sel.targets), sel.mode)) + + # Every GPU module test is registered by a call the parser can see, so a + # source added without a registration reaches the error path. + gpu_dir = os.path.join(root, "unsupported", "test", "GPU") + if os.path.isdir(gpu_dir): + gpu_sources = sorted("unsupported/test/GPU/" + name for name in os.listdir(gpu_dir) + if name.endswith(".cpp")) + unmapped = [rel for rel in gpu_sources if rel not in source_targets] + check(gpu_sources and not unmapped, + "every GPU test source is registered, got %s unmapped of %d" + % (unmapped, len(gpu_sources))) + check(source_targets.get("unsupported/test/GPU/cusolver_svd.cpp") == "cusolver_svd", + "a foreach-registered GPU test maps to its target, got %s" + % source_targets.get("unsupported/test/GPU/cusolver_svd.cpp")) + probe = os.path.join(gpu_dir, "affected_tests_probe.cpp") + with open(probe, "w") as handle: + handle.write('#include "gpu_test_helpers.h"\n') + try: + sel = select(IncludeGraph(root), ["unsupported/test/GPU/affected_tests_probe.cpp"]) + check(sel.mode == "error", + "an unregistered GPU source fails selection, got %s" % sel.mode) + finally: + os.remove(probe) + + # The compile-failure suite must stay reachable: it is filtered out by any + # -R regex that does not name it. + check(len(registered.failtests) > 50, + "real tree registers the failtest suite, got %d" % len(registered.failtests)) + sel = select(graph, sorted(registered.failtests)[:1]) + check(sel.mode == "targets" and sel.ctest_names, + "a changed failtest selects CTest names, got %s (%s)" % (sel.mode, sel.reasons)) + + # main.h is a hub: changing it must run everything. + sel = select(graph, ["test/main.h"]) + check(sel.mode == "all", "test/main.h runs the full suite, got %s" % sel.mode) + + # Private implementation headers may legitimately move. While present, + # they must reach their focused test; broadening to the full suite is safe. + selections = [] + for path, target in (("Eigen/src/Eigenvalues/RealQZ.h", "real_qz"), + ("Eigen/src/SVD/BDCSVD.h", "bdcsvd")): + if path not in graph.files: + print("skipping real-tree check for absent private header %s" % path) + continue + sel = select(graph, [path]) + selections.append(sel) + check(sel.mode in ("targets", "all"), "%s yields safe coverage, got %s" % (path, sel.mode)) + if sel.mode == "targets": + check(target in sel.targets, "%s selects %s" % (path, target)) + + # Every selected name must come from a real CMake registration. + registered_targets = set(source_targets.values()) + for sel in selections: + unknown = sorted(t for t in sel.targets if t not in registered_targets) + check(not unknown, "selected names are test sources, got %s" % unknown) + + +def main(): + root = tempfile.mkdtemp(prefix="eigen-affected-") + try: + build_fixture(root) + test_fixture_graph(root) + test_buildsystem_fixtures(root) + test_failtests(root) + test_cuda_registrations(root) + finally: + shutil.rmtree(root, ignore_errors=True) + test_foreach_registrations() + test_output_encoding() + test_git_rename_paths() + test_real_tree() + + if FAILURES: + print("\n%d check(s) failed" % len(FAILURES)) + return 1 + print("all checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main())