diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h index d40b9fb99..57bfc8cef 100644 --- a/Eigen/src/Core/AssignEvaluator.h +++ b/Eigen/src/Core/AssignEvaluator.h @@ -64,12 +64,27 @@ struct copy_using_evaluator_traits { static constexpr int RestrictedLinearSize = min_size_prefer_fixed(MaxSizeAtCompileTime, MaxPacketSize); static constexpr int OuterStride = outer_stride_at_compile_time::value; - // TODO: distinguish between linear traversal and inner-traversal packet types. - using LinearPacketType = typename find_best_packet::type; + // LinearVectorizedTraversal handles a partial-packet tail (scalar emits under + // complete unrolling, packet_segment under no-unrolling), so we can prefer a + // wider packet than find_best_packet's exact-divisor pick when that strictly + // reduces total op count -- e.g. N=9 float on AVX2 gets 1*Packet8f + 1 scalar + // instead of 2*Packet4f + 1 scalar. find_assign_linear_packet stays on the + // exact-divisor choice when the wider alternative is no better + // (e.g. N=6 double on AVX-512: 3*Packet2d ties with 1*Packet4d + 2 scalars, + // so Packet2d is kept and LLT/LDLT sub-vector kernels are not disturbed). + // InnerVectorizedTraversal still requires Size % PacketSize == 0, so its + // packet type continues to use find_best_packet. + using LinearPacketType = typename find_assign_linear_packet::type; using InnerPacketType = typename find_best_packet::type; static constexpr int LinearPacketSize = unpacket_traits::size; static constexpr int InnerPacketSize = unpacket_traits::size; + // Use the exact-divisor packet size for the Inner-vs-Linear choice below, so + // find_assign_linear_packet's widening only changes the packet *within* + // LinearVectorizedTraversal and never flips an inner-vectorizable assignment + // onto the linear path (e.g. vectorization_logic Matrix57 on NEON int). + static constexpr int LinearTraversalPacketSize = + unpacket_traits::type>::size; public: static constexpr int LinearRequiredAlignment = unpacket_traits::alignment; @@ -84,40 +99,24 @@ struct copy_using_evaluator_traits { (OuterStride != Dynamic) && (OuterStride % InnerPacketSize == 0) && (EIGEN_UNALIGNED_VECTORIZE || JointAlignment >= InnerRequiredAlignment); static constexpr bool MayLinearize = StorageOrdersAgree && (DstFlags & SrcFlags & LinearAccessBit); - static constexpr int CoeffReadCost = int(DstEvaluator::CoeffReadCost) + int(SrcEvaluator::CoeffReadCost); - static constexpr bool SmallAssignmentScalarPathIsCheap = - (SizeAtCompileTime != Dynamic) && (SizeAtCompileTime * CoeffReadCost <= EIGEN_UNROLLING_LIMIT); - /* Packet traversal has enough setup/tail overhead that it is not worth it - for very small fixed-size assignments when the scalar path can be fully - unrolled. More expensive RHS expressions can still amortize packet setup. */ - static constexpr int SmallAssignmentPacketThreshold = 3; - static constexpr int LinearPacketThreshold = SmallAssignmentScalarPathIsCheap ? SmallAssignmentPacketThreshold : 1; - static constexpr int LinearSizeThreshold = LinearPacketThreshold * LinearPacketSize; static constexpr bool MayLinearVectorize = MightVectorize && MayLinearize && DstHasDirectAccess && (EIGEN_UNALIGNED_VECTORIZE || (DstAlignment >= LinearRequiredAlignment) || MaxSizeAtCompileTime == Dynamic) && - (MaxSizeAtCompileTime == Dynamic || MaxSizeAtCompileTime >= LinearSizeThreshold); - /* Slice vectorization can be slow, so use MaxInnerSize rather than InnerSize: - a dynamic block in a fixed-size matrix can still have large slices. With - EIGEN_UNALIGNED_VECTORIZE and unrolling, one packet is still worthwhile for - non-vector slices. Cheap fixed-size vector blocks can otherwise fall back to - slice vectorization after the linear path is rejected, so use the same - conservative cutoff there. */ - static constexpr bool UseConservativeVectorInnerThreshold = IsVectorAtCompileTime && SmallAssignmentScalarPathIsCheap; - static constexpr int VectorInnerPacketThreshold = - (UseConservativeVectorInnerThreshold || !EIGEN_UNALIGNED_VECTORIZE) ? SmallAssignmentPacketThreshold : 1; - static constexpr int VectorInnerSizeThreshold = VectorInnerPacketThreshold * InnerPacketSize; - static constexpr int NonVectorInnerSizeThreshold = - (EIGEN_UNALIGNED_VECTORIZE ? 1 : SmallAssignmentPacketThreshold) * InnerPacketSize; - static constexpr int InnerSizeThreshold = - IsVectorAtCompileTime ? VectorInnerSizeThreshold : NonVectorInnerSizeThreshold; + (MaxSizeAtCompileTime == Dynamic || MaxSizeAtCompileTime >= LinearPacketSize); + /* If the destination isn't aligned, we have to do runtime checks and we don't unroll, + so it's only good for large enough sizes. */ + static constexpr int InnerSizeThreshold = (EIGEN_UNALIGNED_VECTORIZE ? 1 : 3) * InnerPacketSize; static constexpr bool MaySliceVectorize = MightVectorize && DstHasDirectAccess && (MaxInnerSizeAtCompileTime == Dynamic || MaxInnerSizeAtCompileTime >= InnerSizeThreshold); + /* slice vectorization can be slow, so we only want it if the slices are big, which is + indicated by InnerMaxSize rather than InnerSize, think of the case of a dynamic block + in a fixed-size matrix + However, with EIGEN_UNALIGNED_VECTORIZE and unrolling, slice vectorization is still worth it */ public: static constexpr int Traversal = SizeAtCompileTime == 0 ? AllAtOnceTraversal - : (MayLinearVectorize && (LinearPacketSize > InnerPacketSize)) + : (MayLinearVectorize && (LinearTraversalPacketSize > InnerPacketSize)) ? LinearVectorizedTraversal : MayInnerVectorize ? InnerVectorizedTraversal : MayLinearVectorize ? LinearVectorizedTraversal @@ -132,6 +131,7 @@ struct copy_using_evaluator_traits { private: static constexpr int ActualPacketSize = Vectorized ? unpacket_traits::size : 1; static constexpr int UnrollingLimit = EIGEN_UNROLLING_LIMIT * ActualPacketSize; + static constexpr int CoeffReadCost = int(DstEvaluator::CoeffReadCost) + int(SrcEvaluator::CoeffReadCost); static constexpr bool MayUnrollCompletely = (SizeAtCompileTime != Dynamic) && (SizeAtCompileTime * CoeffReadCost <= UnrollingLimit); static constexpr bool MayUnrollInner = @@ -520,11 +520,19 @@ struct dense_assignment_loop_impl::size; static constexpr int Size = Kernel::AssignmentTraits::SizeAtCompileTime; static constexpr int AlignedSize = numext::round_down(Size, PacketSize); - static constexpr bool UsePacketSegment = Kernel::AssignmentTraits::UsePacketSegment; EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) { copy_using_evaluator_linearvec_CompleteUnrolling::run(kernel); - copy_using_evaluator_linearvec_segment::run(kernel); + // Partial-packet tail. Unlike the NoUnrolling case above, here the size is a + // compile-time constant and the loop is fully unrolled, so the tail is a + // fixed handful (fewer than PacketSize) of scalar assignments. Emit those + // directly by forcing UsePacketSegment = false: a masked packet segment + // buys nothing when there is no runtime-variable trip count to collapse, + // and on the AVX backend it lowers to a masked store (vmaskmovps/vmaskmovpd) + // that is slow on current hardware and forwards poorly to later loads. The + // NoUnrolling loop keeps UsePacketSegment because there the tail length is a + // runtime value, where a single masked op does beat a scalar remainder loop. + copy_using_evaluator_linearvec_segment::run(kernel); } }; diff --git a/Eigen/src/Core/arch/SSE/Reductions.h b/Eigen/src/Core/arch/SSE/Reductions.h index 9158ac96d..91dc2ce89 100644 --- a/Eigen/src/Core/arch/SSE/Reductions.h +++ b/Eigen/src/Core/arch/SSE/Reductions.h @@ -219,7 +219,17 @@ struct sse_predux_common { template <> EIGEN_STRONG_INLINE float predux(const Packet4f& a) { +#ifdef EIGEN_VECTORIZE_AVX return sse_predux_impl::run(a); +#else + // See predux(const Packet2d&): on legacy SSE the final 2->1 step is scalar. + Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a, a)); +#ifdef EIGEN_VECTORIZE_SSE3 + return _mm_cvtss_f32(_mm_add_ss(tmp, _mm_movehdup_ps(tmp))); +#else + return _mm_cvtss_f32(_mm_add_ss(tmp, _mm_shuffle_ps(tmp, tmp, 1))); +#endif +#endif } template <> @@ -275,7 +285,17 @@ struct sse_predux_common { template <> EIGEN_STRONG_INLINE double predux(const Packet2d& a) { +#ifdef EIGEN_VECTORIZE_AVX + // With VEX (3-operand) encoding the packed reduction is fine. return sse_predux_impl::run(a); +#else + // Legacy SSE (two-operand) encoding: a packed final reduction step writes a + // live, unused high lane that couples into the dependency graph and + // pessimizes fused kernels with many small reductions (e.g. chained + // fixed-size matrix products) by ~25%. A scalar add produces the same low + // lane without that false coupling. + return _mm_cvtsd_f64(_mm_add_sd(a, _mm_unpackhi_pd(a, a))); +#endif } template <> diff --git a/Eigen/src/Core/util/XprHelper.h b/Eigen/src/Core/util/XprHelper.h index 5d2ce1db2..36eaba8cf 100644 --- a/Eigen/src/Core/util/XprHelper.h +++ b/Eigen/src/Core/util/XprHelper.h @@ -280,6 +280,61 @@ struct find_best_packet { using type = typename find_best_packet_helper::type>::type; }; +// Like find_best_packet, but picks the widest packet whose size is <= Size +// rather than the widest that exactly divides Size. The caller handles any tail. +template = unpacket_traits::size || + std::is_same::half>::value> +struct find_largest_packet_helper; + +template +struct find_largest_packet_helper { + using type = PacketType; +}; + +template +struct find_largest_packet_helper { + using type = typename find_largest_packet_helper::half>::type; +}; + +template +struct find_largest_packet { + using type = typename find_largest_packet_helper::type>::type; +}; + +// Pick the packet type for a linear-traversal assignment: the widest packet +// whose is minimal. +// +// find_best_packet picks the widest exact divisor (no scalar tail). That +// overshoots when no exact divisor exists -- it falls through to the smallest +// packet, e.g. Packet4f at N=9 float on AVX2 emits 2*SSE + 1 scalar where +// 1*Packet8f + 1 scalar would do. We prefer find_largest_packet whenever it +// strictly cuts the op count; otherwise we keep find_best_packet so kernels +// like LLT/LDLT that rely on exact-fit narrow packets are not disturbed +// (e.g. 3*Packet2d == 1*Packet4d + 2 scalars at N=6 double on AVX-512, both +// 3 ops -- keep Packet2d). +// +// Only used in LinearVectorizedTraversal, whose tail handling already accepts +// a partial-packet remainder. InnerVectorized / SliceVectorized still require +// exact divisibility, so they continue to use find_best_packet. +template +struct find_assign_linear_packet { + private: + using best_type = typename find_best_packet::type; + using largest_type = typename find_largest_packet::type; + // Op count = full packets + scalar-tail elements (one scalar emit per tail + // element under CompleteUnrolling). Both helpers return the max packet for + // Dynamic, so the op-count tie there harmlessly resolves to find_best. + template + static constexpr int ops() { + constexpr int sz = unpacket_traits

::size; + return Size == Dynamic ? 0 : Size / sz + Size % sz; + } + + public: + using type = std::conditional_t<(ops() < ops()), largest_type, best_type>; +}; + template ::size) || std::is_same::half>::value> diff --git a/benchmarks/Core/CMakeLists.txt b/benchmarks/Core/CMakeLists.txt index 0bdcbc693..a0dd7dfb0 100644 --- a/benchmarks/Core/CMakeLists.txt +++ b/benchmarks/Core/CMakeLists.txt @@ -25,3 +25,4 @@ eigen_add_benchmark(bench_construction bench_construction.cpp) eigen_add_benchmark(bench_fixed_size bench_fixed_size.cpp) eigen_add_benchmark(bench_fixed_size_double bench_fixed_size.cpp DEFINITIONS SCALAR=double) eigen_add_benchmark(bench_small_matrix bench_small_matrix.cpp) +eigen_add_benchmark(bench_segment_tail bench_segment_tail.cpp) diff --git a/benchmarks/Core/bench_segment_tail.cpp b/benchmarks/Core/bench_segment_tail.cpp new file mode 100644 index 000000000..e0e8e68a5 --- /dev/null +++ b/benchmarks/Core/bench_segment_tail.cpp @@ -0,0 +1,564 @@ +// Benchmarks for the masked partial-packet ("segment") tail of vectorized +// assignments -- the path gated by internal::has_packet_segment. +// See libeigen/eigen#3083 (the regression) and #3086 (segment-tail placement). +// +// Part A measures end-to-end assignments whose length is not a multiple of +// the packet size, so the tail loop is on the hot path: +// * fixed-size -> LinearVectorizedTraversal / CompleteUnrolling +// (compile-time tail count; the #3083 regression shape) +// * dynamic -> LinearVectorizedTraversal / NoUnrolling +// (runtime tail count) +// Part A's destinations are not read back, so it does NOT exercise the +// store-to-load forwarding hazard -- see Part C. +// +// Part B is a direct A/B of the segment primitives (ploaduSegment / +// pstoreuSegment) against an equivalent scalar loop, swept over the tail +// length. This isolates the masked load/store hardware cost (e.g. AVX2 +// vmaskmov) and the store-to-load forwarding stall, and is the data that +// drives the per-ISA has_packet_segment decision. +// +// Part C runs the #3083 regression kernels: chained small fixed-size matrix +// ops where each result is consumed by the next. A partial-packet tail store +// then collides with the consumer's packet load -- the store-to-load +// forwarding hazard that makes a masked tail far more costly than a scalar +// one. This is the cost Parts A and B cannot see in isolation. +// +// The active packet size and the value of has_packet_segment per scalar type +// are emitted as Google Benchmark custom context, so a captured run is +// self-describing. To sweep ISAs, build once per target, e.g.: +// g++ -O3 -DNDEBUG -std=c++17 -msse4.2 bench_segment_tail.cpp ... +// g++ -O3 -DNDEBUG -std=c++17 -mavx bench_segment_tail.cpp ... +// g++ -O3 -DNDEBUG -std=c++17 -mavx2 -mfma bench_segment_tail.cpp ... +// g++ -O3 -DNDEBUG -std=c++17 -march=native bench_segment_tail.cpp ... +// or, via CMake, configure benchmarks/ with -DCMAKE_CXX_FLAGS=. +// +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include + +#include +#include +#include +#include + +#include +#include + +using namespace Eigen; + +namespace { + +// Capacity of the Part B scratch buffers (Buffers). Must be >= the largest +// packet size of any benchmarked scalar type, since the primitives load and +// store a full packet; the static_assert in Buffers enforces it. +constexpr int kMaxCount = 16; + +// --------------------------------------------------------------------------- +// Records, per scalar type, the packet size in use and the value of +// has_packet_segment for the active ISA. Emitted as Google Benchmark custom +// context so it appears once in the report header and in any --benchmark_out +// JSON, making the captured baseline self-describing. +// --------------------------------------------------------------------------- +template +void add_trait_context(const char* name) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + constexpr bool HasSegment = internal::has_packet_segment::value; + benchmark::AddCustomContext( + std::string("segment.") + name, + "PacketSize=" + std::to_string(PacketSize) + " has_packet_segment=" + (HasSegment ? "true" : "false")); +} + +// =========================================================================== +// Part A : assignment-level tail benchmarks +// =========================================================================== + +// Fixed-size column vector: dst = a + b, with N a compile-time constant. +// When N % PacketSize != 0 the completely-unrolled assignment ends in a +// compile-time-length segment tail. +template +void BM_FixedAssign(benchmark::State& state) { + using Vec = Matrix; + Vec a = Vec::Random(); + Vec b = Vec::Random(); + Vec dst = Vec::Random(); + for (auto _ : state) { + benchmark::DoNotOptimize(a.data()); + benchmark::DoNotOptimize(b.data()); + dst = a + b; + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(state.iterations() * static_cast(N) * sizeof(T) * 3); +} + +// Dynamic-size column vector: dst = a + b, with a runtime length that is not a +// multiple of the packet size, so the NoUnrolling tail loop runs every call. +template +void BM_DynamicAssign(benchmark::State& state) { + const Index size = state.range(0); + Matrix a = Matrix::Random(size); + Matrix b = Matrix::Random(size); + Matrix dst = Matrix::Random(size); + for (auto _ : state) { + benchmark::DoNotOptimize(a.data()); + benchmark::DoNotOptimize(b.data()); + dst = a + b; + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(state.iterations() * static_cast(size) * sizeof(T) * 3); +} + +// =========================================================================== +// Part B : segment-primitive A/B micro-benchmarks +// +// Each pair (segment vs scalar) is registered over the same count range so the +// reports line up. count is taken from state.range(0): runtime for the +// non-CT variants (matches the dynamic tail), compile-time for the *_CT +// variants (matches the completely-unrolled tail). +// =========================================================================== + +template +struct Buffers { + static_assert(kMaxCount >= internal::unpacket_traits::type>::size, + "kMaxCount is smaller than this type's packet size: the scratch buffers cannot hold a full packet"); + EIGEN_ALIGN_MAX T src[kMaxCount]; + EIGEN_ALIGN_MAX T dst[kMaxCount]; + Buffers() { + for (int i = 0; i < kMaxCount; ++i) { + src[i] = T(i + 1); + dst[i] = T(0); + } + } +}; + +// --- runtime-count store: masked segment store vs scalar store loop --------- +template +void BM_PrimSegmentStore(benchmark::State& state) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + const Index count = state.range(0); + if (PacketSize <= 1 || count > PacketSize) { + state.SkipWithError("count exceeds packet size"); + return; + } + Buffers buf; + Packet p = internal::ploadu(buf.src); + for (auto _ : state) { + benchmark::DoNotOptimize(p); + internal::pstoreuSegment(buf.dst, p, 0, count); + benchmark::DoNotOptimize(buf.dst); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * count); +} + +template +void BM_PrimScalarStore(benchmark::State& state) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + const Index count = state.range(0); + if (PacketSize <= 1 || count > PacketSize) { + state.SkipWithError("count exceeds packet size"); + return; + } + Buffers buf; + for (auto _ : state) { + benchmark::DoNotOptimize(buf.src); + for (Index k = 0; k < count; ++k) buf.dst[k] = buf.src[k]; + benchmark::DoNotOptimize(buf.dst); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * count); +} + +// --- runtime-count store+reload: captures store-to-load forwarding stalls --- +template +void BM_PrimSegmentStoreReload(benchmark::State& state) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + const Index count = state.range(0); + if (PacketSize <= 1 || count > PacketSize) { + state.SkipWithError("count exceeds packet size"); + return; + } + Buffers buf; + Packet p = internal::ploadu(buf.src); + Packet acc = internal::pset1(T(0)); + for (auto _ : state) { + benchmark::DoNotOptimize(p); + internal::pstoreuSegment(buf.dst, p, 0, count); + benchmark::DoNotOptimize(buf.dst); + Packet q = internal::ploaduSegment(buf.dst, 0, count); + acc = internal::padd(acc, q); + benchmark::DoNotOptimize(acc); + } + state.SetItemsProcessed(state.iterations() * count); +} + +template +void BM_PrimScalarStoreReload(benchmark::State& state) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + const Index count = state.range(0); + if (PacketSize <= 1 || count > PacketSize) { + state.SkipWithError("count exceeds packet size"); + return; + } + Buffers buf; + for (auto _ : state) { + benchmark::DoNotOptimize(buf.src); + for (Index k = 0; k < count; ++k) buf.dst[k] = buf.src[k]; + benchmark::DoNotOptimize(buf.dst); + // acc is reset every iteration: a persistent accumulator would grow without + // bound over the benchmark's iterations -- signed-integer overflow (UB) for + // integer T. Reset keeps the reload + add live without that. + T acc = T(0); + for (Index k = 0; k < count; ++k) acc += buf.dst[k]; + benchmark::DoNotOptimize(acc); + } + state.SetItemsProcessed(state.iterations() * count); +} + +// --- overlapping full-width store+reload: one unmasked packet, count-free --- +// Models the "store a full packet ending at size" tail strategy: valid for a +// plain assignment when size >= PacketSize. Mask-free, so it is available on +// every ISA (including SSE/NEON), and a full-width store forwards cleanly to a +// same-width reload -- unlike a masked store. Cost is independent of the tail +// length, so this is registered once per type rather than swept over count. +template +void BM_PrimOverlapStoreReload(benchmark::State& state) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + if (PacketSize <= 1) { + state.SkipWithError("type not vectorized"); + return; + } + Buffers buf; + Packet p = internal::ploadu(buf.src); + Packet acc = internal::pset1(T(0)); + for (auto _ : state) { + benchmark::DoNotOptimize(p); + internal::pstoreu(buf.dst, p); + benchmark::DoNotOptimize(buf.dst); + Packet q = internal::ploadu(buf.dst); + acc = internal::padd(acc, q); + benchmark::DoNotOptimize(acc); + } + state.SetItemsProcessed(state.iterations() * PacketSize); +} + +// --- runtime-count load: masked segment load vs scalar load loop ----------- +template +void BM_PrimSegmentLoad(benchmark::State& state) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + const Index count = state.range(0); + if (PacketSize <= 1 || count > PacketSize) { + state.SkipWithError("count exceeds packet size"); + return; + } + Buffers buf; + Packet acc = internal::pset1(T(0)); + for (auto _ : state) { + benchmark::DoNotOptimize(buf.src); + Packet q = internal::ploaduSegment(buf.src, 0, count); + acc = internal::padd(acc, q); + benchmark::DoNotOptimize(acc); + } + state.SetItemsProcessed(state.iterations() * count); +} + +template +void BM_PrimScalarLoad(benchmark::State& state) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + const Index count = state.range(0); + if (PacketSize <= 1 || count > PacketSize) { + state.SkipWithError("count exceeds packet size"); + return; + } + Buffers buf; + for (auto _ : state) { + benchmark::DoNotOptimize(buf.src); + // acc is reset every iteration -- see BM_PrimScalarStoreReload. + T acc = T(0); + for (Index k = 0; k < count; ++k) acc += buf.src[k]; + benchmark::DoNotOptimize(acc); + } + state.SetItemsProcessed(state.iterations() * count); +} + +// --- compile-time-count store: matches the completely-unrolled tail -------- +// Here the scalar alternative is itself fully unrolled (no loop), which is why +// it can beat a masked store for tiny tails. +template +void BM_PrimSegmentStoreCT(benchmark::State& state) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + if (PacketSize <= 1 || Count > PacketSize) { + state.SkipWithError("count exceeds packet size"); + return; + } + Buffers buf; + Packet p = internal::ploadu(buf.src); + for (auto _ : state) { + benchmark::DoNotOptimize(p); + internal::pstoreuSegment(buf.dst, p, 0, Count); + benchmark::DoNotOptimize(buf.dst); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * Count); +} + +template +void BM_PrimScalarStoreCT(benchmark::State& state) { + using Packet = typename internal::packet_traits::type; + constexpr int PacketSize = internal::unpacket_traits::size; + if (PacketSize <= 1 || Count > PacketSize) { + state.SkipWithError("count exceeds packet size"); + return; + } + Buffers buf; + for (auto _ : state) { + benchmark::DoNotOptimize(buf.src); + EIGEN_UNROLL_LOOP + for (int k = 0; k < Count; ++k) buf.dst[k] = buf.src[k]; + benchmark::DoNotOptimize(buf.dst); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * Count); +} + +// =========================================================================== +// Part C : chained fixed-size kernels (results consumed immediately) +// +// The #3083 regression kernels. Unlike Part A, each assignment's result feeds +// the next op, so a partial-packet tail store is followed by a packet load of +// the same destination. That store-to-load forwarding hazard is what makes a +// masked (or overlapping) tail far more costly than a scalar one -- it is only +// visible when the result is consumed, which these benchmarks do and Part A +// deliberately does not. These are the kernels that reflect the real cost. +// =========================================================================== + +// #3083 Example 2: double 3x3 inverse() * t -- the headline regression. +EIGEN_DONT_INLINE void kernel_inverse(const Vector3d& a, const Vector3d& b, const Vector3d& c, const Matrix3d& R, + const Vector3d& t, Vector3d& result) { + Matrix3d A; + A << a, -R * b, c; + result = A.inverse() * t; +} + +void BM_Chained_Inverse3x3(benchmark::State& state) { + Vector3d a = Vector3d::Random(), b = Vector3d::Random(), c = Vector3d::Random(), t = Vector3d::Random(), result; + Matrix3d R = Matrix3d::Random(); + for (auto _ : state) { + benchmark::DoNotOptimize(a); + benchmark::DoNotOptimize(b); + benchmark::DoNotOptimize(R); + kernel_inverse(a, b, c, R, t, result); + benchmark::DoNotOptimize(result); + benchmark::ClobberMemory(); + } +} + +// #3083 Example 1: camera kernel chaining Matrix blocks. +using Blk23 = Matrix; +using Blocks5 = std::tuple; + +EIGEN_DONT_INLINE void kernel_camera(const Matrix3d& A, const Matrix3d& B, const Vector3d& u, const Vector3d& v, + const Vector3d& w, const Vector3d& x, const Vector4d& params, Vector2d& output, + Blocks5& blocks) { + const Vector3d a = A * u + v; + const Vector3d b = B.transpose() * (a - w); + const Vector3d c = b - x; + const double iz = 1.0 / c.z(); + const double iz2 = iz * iz; + const double k0 = params(0), k1 = params(1); + Blk23 J; + J << k0 * iz, 0.0, -k0 * c.x() * iz2, 0.0, k1 * iz, -k1 * c.y() * iz2; + std::get<0>(blocks) = J * B.transpose(); + std::get<1>(blocks) = std::get<0>(blocks) * A; + std::get<2>(blocks) = -std::get<1>(blocks); + std::get<3>(blocks) = std::get<0>(blocks) + std::get<1>(blocks); + std::get<4>(blocks) = std::get<3>(blocks) - std::get<2>(blocks); + output << k0 * c.x() * iz + params(2), k1 * c.y() * iz + params(3); +} + +void BM_Chained_Camera(benchmark::State& state) { + Matrix3d A = Matrix3d::Random(), B = Matrix3d::Random(); + Vector3d u = Vector3d::Random(), v = Vector3d::Random(), w = Vector3d::Random(), x = Vector3d::Random(); + Vector4d params = Vector4d::Random(); + Vector2d output; + Blocks5 blocks; + for (auto _ : state) { + benchmark::DoNotOptimize(A); + benchmark::DoNotOptimize(B); + benchmark::DoNotOptimize(u); + kernel_camera(A, B, u, v, w, x, params, output, blocks); + benchmark::DoNotOptimize(&blocks); + benchmark::DoNotOptimize(output); + benchmark::ClobberMemory(); + } +} + +// Direct chained Matrix sums: isolates the 6-element tail with a +// read-after-write dependency between consecutive assignments. +void BM_Chained_Block23(benchmark::State& state) { + Blk23 m0 = Blk23::Random(), m1 = Blk23::Random(), m2 = Blk23::Random(), m3 = Blk23::Random(); + for (auto _ : state) { + benchmark::DoNotOptimize(m0); + m1 = m0 + m1; + m2 = m1 + m2; + m3 = m2 + m3; + benchmark::DoNotOptimize(m3); + benchmark::ClobberMemory(); + } +} + +// =========================================================================== +// Registration +// +// Done programmatically (rather than via the BENCHMARK macro) so that Part B's +// tail count is clamped to the packet size actually in use: the report then +// contains no inapplicable count > PacketSize entries, whatever ISA the source +// was compiled for. +// =========================================================================== + +template +const char* type_tag(); +template <> +const char* type_tag() { + return "f32"; +} +template <> +const char* type_tag() { + return "f64"; +} +template <> +const char* type_tag() { + return "i32"; +} +template <> +const char* type_tag() { + return "i64"; +} +template <> +const char* type_tag>() { + return "cf32"; +} +template <> +const char* type_tag>() { + return "cf64"; +} + +template +int packet_size() { + return internal::unpacket_traits::type>::size; +} + +// Part A : fixed-size tails. Exact multiples (8/16/32/64) act as no-tail +// controls; the rest cover a spread of remainders across SSE/AVX/AVX-512. +template +void add_fixed() { + benchmark::RegisterBenchmark(std::string("FixedAssign/") + type_tag() + "/" + std::to_string(N), + &BM_FixedAssign); +} + +template +void add_all_fixed() { + add_fixed(); + add_fixed(); + add_fixed(); + add_fixed(); + add_fixed(); + add_fixed(); + add_fixed(); + add_fixed(); + add_fixed(); + add_fixed(); + add_fixed(); + add_fixed(); +} + +// Part A : dynamic-size tails. Small sizes keep the tail a meaningful fraction; +// 1025 confirms there is no regression once the tail is amortized. +template +void add_dynamic() { + auto* b = benchmark::RegisterBenchmark(std::string("DynAssign/") + type_tag(), &BM_DynamicAssign); + for (int s : {15, 17, 31, 33, 63, 65, 127, 129, 255, 1025}) b->Arg(s); +} + +// Part B : segment vs scalar primitives, runtime count 1..PacketSize. +void add_prim_range(const std::string& name, void (*fn)(benchmark::State&), int ps) { + auto* b = benchmark::RegisterBenchmark(name, fn); + for (int c = 1; c <= ps; ++c) b->Arg(c); +} + +template +void add_prim() { + const int ps = packet_size(); + if (ps <= 1) return; // type not vectorized on this ISA + const std::string tag = type_tag(); + add_prim_range("Prim/Store/seg/" + tag, &BM_PrimSegmentStore, ps); + add_prim_range("Prim/Store/scl/" + tag, &BM_PrimScalarStore, ps); + add_prim_range("Prim/StoreReload/seg/" + tag, &BM_PrimSegmentStoreReload, ps); + add_prim_range("Prim/StoreReload/scl/" + tag, &BM_PrimScalarStoreReload, ps); + benchmark::RegisterBenchmark("Prim/StoreReload/ovl/" + tag, &BM_PrimOverlapStoreReload); + add_prim_range("Prim/Load/seg/" + tag, &BM_PrimSegmentLoad, ps); + add_prim_range("Prim/Load/scl/" + tag, &BM_PrimScalarLoad, ps); +} + +// Part B : compile-time count store (matches the completely-unrolled tail). +template +void add_prim_ct() { + if (Count > packet_size()) return; + const std::string tag = std::string(type_tag()) + "/" + std::to_string(Count); + benchmark::RegisterBenchmark("PrimCT/Store/seg/" + tag, &BM_PrimSegmentStoreCT); + benchmark::RegisterBenchmark("PrimCT/Store/scl/" + tag, &BM_PrimScalarStoreCT); +} + +template +void add_all_prim_ct() { + add_prim_ct(); + add_prim_ct(); + add_prim_ct(); + add_prim_ct(); +} + +int RegisterAll() { + add_trait_context("f32"); + add_trait_context("f64"); + add_trait_context("i32"); + add_trait_context("i64"); + add_trait_context>("cf32"); + add_trait_context>("cf64"); + + add_all_fixed(); + add_all_fixed(); + add_dynamic(); + add_dynamic(); + + add_prim(); + add_prim(); + add_prim(); + add_prim(); + add_prim>(); + add_prim>(); + + add_all_prim_ct(); + add_all_prim_ct(); + + benchmark::RegisterBenchmark("Chained/Inverse3x3", &BM_Chained_Inverse3x3); + benchmark::RegisterBenchmark("Chained/Camera", &BM_Chained_Camera); + benchmark::RegisterBenchmark("Chained/Block23", &BM_Chained_Block23); + return 0; +} + +const int registered = RegisterAll(); + +} // namespace + +// main() is provided by benchmark::benchmark_main (linked via eigen_add_benchmark). diff --git a/test/vectorization_logic.cpp b/test/vectorization_logic.cpp index e0358bfb3..523389108 100644 --- a/test/vectorization_logic.cpp +++ b/test/vectorization_logic.cpp @@ -85,56 +85,6 @@ bool test_assign(int traversal, int unrolling) { return res; } -template -bool test_add_assign(int traversal, int unrolling) { - EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst, Src); - typedef internal::copy_using_evaluator_traits, internal::evaluator, - internal::add_assign_op > - traits; - bool res = traits::Traversal == traversal && traits::Unrolling == unrolling; - if (!res) { - std::cerr << "Src: " << demangle_flags(Src::Flags) << std::endl; - std::cerr << " " << demangle_flags(internal::evaluator::Flags) << std::endl; - std::cerr << "Dst: " << demangle_flags(Dst::Flags) << std::endl; - std::cerr << " " << demangle_flags(internal::evaluator::Flags) << std::endl; - traits::debug(); - std::cerr << " Expected Traversal == " << demangle_traversal(traversal) << " got " - << demangle_traversal(traits::Traversal) << "\n"; - std::cerr << " Expected Unrolling == " << demangle_unrolling(unrolling) << " got " - << demangle_unrolling(traits::Unrolling) << "\n"; - } - return res; -} - -template -bool test_add_assign(const Dst&, const Src&, int traversal, int unrolling) { - EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst, Src); - typedef internal::copy_using_evaluator_traits, internal::evaluator, - internal::add_assign_op > - traits; - // If traversal or unrolling are negative, ignore. - bool res = traversal > -1 ? traits::Traversal == traversal : true; - if (unrolling > -1) { - if (unrolling == InnerUnrolling + CompleteUnrolling) { - res = res && (int(traits::Unrolling) == InnerUnrolling || int(traits::Unrolling) == CompleteUnrolling); - } else { - res = res && int(traits::Unrolling) == unrolling; - } - } - if (!res) { - std::cerr << "Src: " << demangle_flags(Src::Flags) << std::endl; - std::cerr << " " << demangle_flags(internal::evaluator::Flags) << std::endl; - std::cerr << "Dst: " << demangle_flags(Dst::Flags) << std::endl; - std::cerr << " " << demangle_flags(internal::evaluator::Flags) << std::endl; - traits::debug(); - std::cerr << " Expected Traversal == " << demangle_traversal(traversal) << " got " - << demangle_traversal(traits::Traversal) << "\n"; - std::cerr << " Expected Unrolling == " << demangle_unrolling(unrolling) << " got " - << demangle_unrolling(traits::Unrolling) << "\n"; - } - return res; -} - template bool test_redux(const Xpr&, int traversal, int unrolling) { typedef typename Xpr::Scalar Scalar; @@ -208,9 +158,6 @@ struct vectorization_logic { : /*PacketSize==1 ?*/ 1), DontAlign | ((Matrix1::Flags & RowMajorBit) ? RowMajor : ColMajor)> Matrix1u; - constexpr int Matrix1InnerPacketSize = internal::unpacket_traits< - typename internal::find_best_packet::type>::size; - constexpr bool Matrix1CanInnerVectorize = int(Matrix1::InnerSizeAtCompileTime) % Matrix1InnerPacketSize == 0; // this type is made such that it can only be vectorized when viewed as a linear 1D vector typedef Matrix 1) { - typedef Matrix Vector5; - typedef Matrix Vector8; - typedef Block Vector8Block5; - typedef Block Vector8Block3; - enum { - Vector5PacketSize = internal::unpacket_traits::type>::size, - Vector3PacketSize = internal::unpacket_traits::type>::size - }; - enum { - ThresholdPacketSize = internal::unpacket_traits::type>::size - }; - typedef Matrix Vector2Packets; - typedef Matrix VectorAlmost3Packets; - typedef Matrix Vector3Packets; - typedef Matrix Vector3PacketsPlus1; - typedef Block VectorBlock2Packets; - typedef Block VectorBlockAlmost3Packets; - typedef Block VectorBlock3Packets; - if (Vector5PacketSize > 1) { - VERIFY((test_assign(LinearTraversal, CompleteUnrolling))); - VERIFY((test_add_assign(LinearTraversal, CompleteUnrolling))); - VERIFY((test_add_assign(LinearTraversal, CompleteUnrolling))); - } - if (Vector3PacketSize > 1) { - VERIFY((test_assign(LinearTraversal, CompleteUnrolling))); - VERIFY((test_add_assign(LinearTraversal, CompleteUnrolling))); - } - if (ThresholdPacketSize > 1) { - VERIFY((test_assign(LinearTraversal, CompleteUnrolling))); - VERIFY((test_add_assign(LinearTraversal, CompleteUnrolling))); - VERIFY((test_assign(LinearTraversal, CompleteUnrolling))); - VERIFY((test_add_assign(LinearTraversal, CompleteUnrolling))); - VERIFY((test_assign(LinearTraversal, CompleteUnrolling))); - VERIFY((test_add_assign(LinearTraversal, CompleteUnrolling))); - VERIFY((test_assign( - EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal, CompleteUnrolling))); - VERIFY((test_add_assign( - EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal, CompleteUnrolling))); - VERIFY((test_assign( - EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearTraversal, CompleteUnrolling))); - VERIFY((test_add_assign( - EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearTraversal, CompleteUnrolling))); - Vector3PacketsPlus1 vector3_packets_plus1; - VERIFY(test_add_assign( - vector3_packets_plus1.template segment<2 * ThresholdPacketSize>(1).array(), - vector3_packets_plus1.template tail<2 * ThresholdPacketSize>().array().log(), - (EIGEN_UNALIGNED_VECTORIZE && PacketTraits::HasLog) ? LinearVectorizedTraversal : LinearTraversal, -1)); - } - typedef Matrix Matrix33c; typedef Matrix Vector3; VERIFY( @@ -426,12 +327,6 @@ struct vectorization_logic_half { typedef Matrix Matrix57; typedef Matrix Matrix35; typedef Matrix Matrix57u; - constexpr int Vector1LinearPacketSize = - internal::unpacket_traits::type>::size; - constexpr bool Vector1CanLinearVectorize = int(Vector1::SizeAtCompileTime) >= 3 * Vector1LinearPacketSize; - constexpr int Vector1SegmentTraversal = - EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal - : (Vector1CanLinearVectorize ? LinearVectorizedTraversal : LinearTraversal); typedef Matrix Matrix1u; - constexpr int Matrix1InnerPacketSize = internal::unpacket_traits< - typename internal::find_best_packet::type>::size; - constexpr bool Matrix1CanInnerVectorize = int(Matrix1::InnerSizeAtCompileTime) % Matrix1InnerPacketSize == 0; // this type is made such that it can only be vectorized when viewed as a linear 1D vector typedef Matrix(0).derived(), Vector1SegmentTraversal, + VERIFY(test_assign(Vector1(), Vector1().template segment(0).derived(), + EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearVectorizedTraversal, CompleteUnrolling)); VERIFY(test_assign(Vector1(), Scalar(RealScalar(2.1)) * Vector1() - Vector1(), InnerVectorizedTraversal, CompleteUnrolling)); @@ -488,7 +381,7 @@ struct vectorization_logic_half { Vector1(), (Scalar(RealScalar(2.1)) * Vector1().template segment(0) - Vector1().template segment(0)) .derived(), - Vector1SegmentTraversal, CompleteUnrolling)); + EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearVectorizedTraversal, CompleteUnrolling)); VERIFY(test_assign(Vector1(), Vector1().cwiseProduct(Vector1()), InnerVectorizedTraversal, CompleteUnrolling)); VERIFY(test_assign(Vector1(), Vector1().template cast(), InnerVectorizedTraversal, CompleteUnrolling)); @@ -500,7 +393,8 @@ struct vectorization_logic_half { VERIFY(test_assign(Matrix1u(), Matrix1() + Matrix1(), EIGEN_UNALIGNED_VECTORIZE - ? (Matrix1CanInnerVectorize ? InnerVectorizedTraversal : LinearTraversal) + ? ((int(Matrix1::InnerSizeAtCompileTime) % int(PacketSize)) == 0 ? InnerVectorizedTraversal + : LinearVectorizedTraversal) : LinearTraversal, CompleteUnrolling)); @@ -579,15 +473,15 @@ EIGEN_DECLARE_TEST(vectorization_logic) { CALL_SUBTEST(vectorization_logic_half >::run()); // For backends without sub-packet types (e.g. the generic clang backend), - // find_best_packet may return a packet too large for tiny fixed-size assignments, - // making MayLinearVectorize false in the assignment path (which requires - // SizeAtCompileTime >= 3 * PacketSize). The redux path has no such constraint. + // find_best_packet may return a packet larger than the matrix, making + // MayLinearVectorize false in the assignment path (which requires + // SizeAtCompileTime >= PacketSize). The redux path has no such constraint. if (internal::packet_traits::Vectorizable) { constexpr int kFloatBestPacketSize3x3 = internal::unpacket_traits::type>::size; VERIFY(test_assign( Matrix(), Matrix() + Matrix(), - EIGEN_UNALIGNED_VECTORIZE && 3 * kFloatBestPacketSize3x3 <= 9 ? LinearVectorizedTraversal : LinearTraversal, + EIGEN_UNALIGNED_VECTORIZE && kFloatBestPacketSize3x3 <= 9 ? LinearVectorizedTraversal : LinearTraversal, CompleteUnrolling)); VERIFY(test_redux(Matrix(), EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal, @@ -599,7 +493,7 @@ EIGEN_DECLARE_TEST(vectorization_logic) { internal::unpacket_traits::type>::size; VERIFY(test_assign( Matrix(), Matrix() + Matrix(), - EIGEN_UNALIGNED_VECTORIZE && 3 * kDoubleBestPacketSize3x3 <= 9 ? LinearVectorizedTraversal : LinearTraversal, + EIGEN_UNALIGNED_VECTORIZE && kDoubleBestPacketSize3x3 <= 9 ? LinearVectorizedTraversal : LinearTraversal, CompleteUnrolling)); VERIFY(test_redux(Matrix(), EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal,