diff --git a/Eigen/src/Core/ConcatOp.h b/Eigen/src/Core/ConcatOp.h index 1e73c18d6..91296a6e8 100644 --- a/Eigen/src/Core/ConcatOp.h +++ b/Eigen/src/Core/ConcatOp.h @@ -270,7 +270,7 @@ struct evaluator> : evaluator_base EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetBoundary(Index row, Index col) const { constexpr int packetSize = unpacket_traits::size; - EIGEN_ALIGN_MAX Scalar tmp[packetSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar tmp[packetSize]; for (int i = 0; i < packetSize; ++i) tmp[i] = coeff(row + (Direction == Vertical ? i : 0), col + (Direction == Horizontal ? i : 0)); return pload(tmp); @@ -279,7 +279,7 @@ struct evaluator> : evaluator_base EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetBoundaryLinear(Index index) const { constexpr int packetSize = unpacket_traits::size; - EIGEN_ALIGN_MAX Scalar tmp[packetSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar tmp[packetSize]; for (int i = 0; i < packetSize; ++i) tmp[i] = coeff(index + i); return pload(tmp); } diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index 022ef5f04..e0cfbbe0b 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h @@ -786,7 +786,7 @@ EIGEN_DEVICE_FUNC inline Packet pload_partial(const typename unpacket_traits::size; eigen_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet"); using Scalar = typename unpacket_traits::type; - EIGEN_ALIGN_MAX Scalar elements[packet_size] = {Scalar(0)}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar elements[packet_size] = {Scalar(0)}; for (Index i = offset; i < numext::mini(n + offset, packet_size); i++) { elements[i] = from[i - offset]; } @@ -807,7 +807,7 @@ EIGEN_DEVICE_FUNC inline Packet ploadu_partial(const typename unpacket_traits::size; eigen_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet"); using Scalar = typename unpacket_traits::type; - EIGEN_ALIGN_MAX Scalar elements[packet_size] = {Scalar(0)}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar elements[packet_size] = {Scalar(0)}; for (Index i = offset; i < numext::mini(n + offset, packet_size); i++) { elements[i] = from[i - offset]; } @@ -1064,7 +1064,7 @@ template EIGEN_DEVICE_FUNC inline void pstore_partial(Scalar* to, const Packet& from, const Index n, const Index offset = 0) { const Index packet_size = unpacket_traits::size; eigen_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet"); - EIGEN_ALIGN_MAX Scalar elements[packet_size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar elements[packet_size]; pstore(elements, from); for (Index i = 0; i < numext::mini(n, packet_size - offset); i++) { to[i] = elements[i + offset]; @@ -1082,7 +1082,7 @@ template EIGEN_DEVICE_FUNC inline void pstoreu_partial(Scalar* to, const Packet& from, const Index n, const Index offset = 0) { const Index packet_size = unpacket_traits::size; eigen_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet"); - EIGEN_ALIGN_MAX Scalar elements[packet_size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar elements[packet_size]; pstore(elements, from); for (Index i = 0; i < numext::mini(n, packet_size - offset); i++) { to[i] = elements[i + offset]; @@ -1105,7 +1105,7 @@ EIGEN_DEVICE_FUNC inline Packet pgather(const Scalar* from, Index /*stride*/) { template EIGEN_DEVICE_FUNC inline Packet pgather_partial(const Scalar* from, Index stride, const Index n) { const Index packet_size = unpacket_traits::size; - EIGEN_ALIGN_MAX Scalar elements[packet_size] = {Scalar(0)}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar elements[packet_size] = {Scalar(0)}; for (Index i = 0; i < numext::mini(n, packet_size); i++) { elements[i] = from[i * stride]; } @@ -1120,7 +1120,7 @@ EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index /*s template EIGEN_DEVICE_FUNC inline void pscatter_partial(Scalar* to, const Packet& from, Index stride, const Index n) { const Index packet_size = unpacket_traits::size; - EIGEN_ALIGN_MAX Scalar elements[packet_size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar elements[packet_size]; pstore(elements, from); for (Index i = 0; i < numext::mini(n, packet_size); i++) { to[i * stride] = elements[i]; @@ -1481,7 +1481,7 @@ template EIGEN_DEVICE_FUNC inline typename unpacket_traits::type predux_helper(const Packet& a, Op op) { using Scalar = typename unpacket_traits::type; const size_t n = unpacket_traits::size; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar elements[n]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar elements[n]; pstoreu(elements, a); for (size_t k = n / 2; k > 0; k /= 2) { for (size_t i = 0; i < k; ++i) { diff --git a/Eigen/src/Core/arch/AltiVec/TypeCasting.h b/Eigen/src/Core/arch/AltiVec/TypeCasting.h index 44edbf92d..ca640df51 100644 --- a/Eigen/src/Core/arch/AltiVec/TypeCasting.h +++ b/Eigen/src/Core/arch/AltiVec/TypeCasting.h @@ -132,7 +132,7 @@ EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) #ifdef EIGEN_VECTORIZE_VSX template <> inline Packet2l pcast(const Packet2d& x) { - EIGEN_ALIGN_MAX double dtmp[2]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) double dtmp[2]; pstore(dtmp, x); EIGEN_ALIGN_MAX long long itmp[2] = {static_cast(dtmp[0]), static_cast(dtmp[1])}; return vec_xl(0, itmp); @@ -142,7 +142,8 @@ template <> inline Packet2d pcast(const Packet2l& x) { EIGEN_ALIGN_MAX long long itmp[2]; vec_xst(x, 0, itmp); - EIGEN_ALIGN_MAX double dtmp[2] = {static_cast(itmp[0]), static_cast(itmp[1])}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) + double dtmp[2] = {static_cast(itmp[0]), static_cast(itmp[1])}; return pload(dtmp); } #endif diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h b/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h index e564b8ad1..3949bc8c0 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathTrig.h @@ -149,9 +149,9 @@ EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS if (predux_any(pcmp_le(pset1(huge_th), pabs(_x)))) { const int PacketSize = unpacket_traits::size; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float vals[PacketSize]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) float x_cpy[PacketSize]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Eigen::numext::int32_t y_int2[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) float vals[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) float x_cpy[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Eigen::numext::int32_t y_int2[PacketSize]; pstoreu(vals, pabs(_x)); pstoreu(x_cpy, x); pstoreu(y_int2, y_int); @@ -394,8 +394,8 @@ EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS // for what is in practice a rare path), so these inputs fall back to the scalar libm. if (EIGEN_PREDICT_FALSE(predux_any(pcmp_le(pset1(huge_th), x_abs)))) { const int PacketSize = unpacket_traits::size; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) double sincos_vals[PacketSize]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) double x_cpy[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) double sincos_vals[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) double x_cpy[PacketSize]; pstoreu(x_cpy, x); pstoreu(sincos_vals, sFinalRes); for (int k = 0; k < PacketSize; ++k) { diff --git a/Eigen/src/Core/arch/NEON/TypeCasting.h b/Eigen/src/Core/arch/NEON/TypeCasting.h index f352a56db..06c5def68 100644 --- a/Eigen/src/Core/arch/NEON/TypeCasting.h +++ b/Eigen/src/Core/arch/NEON/TypeCasting.h @@ -1181,11 +1181,12 @@ EIGEN_STRONG_INLINE Packet4f pcast(const Packet2l& a, const #if EIGEN_ARCH_ARM64 return vcombine_f32(vcvt_f32_f64(vcvtq_f64_s64(a)), vcvt_f32_f64(vcvtq_f64_s64(b))); #else - EIGEN_ALIGN_MAX int64_t lvals[4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) int64_t lvals[4]; pstore(lvals, a); pstore(lvals + 2, b); - EIGEN_ALIGN_MAX float fvals[4] = {static_cast(lvals[0]), static_cast(lvals[1]), - static_cast(lvals[2]), static_cast(lvals[3])}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) + float fvals[4] = {static_cast(lvals[0]), static_cast(lvals[1]), static_cast(lvals[2]), + static_cast(lvals[3])}; return pload(fvals); #endif } @@ -1195,9 +1196,10 @@ EIGEN_STRONG_INLINE Packet2f pcast(const Packet2l& a) { #if EIGEN_ARCH_ARM64 return vcvt_f32_f64(vcvtq_f64_s64(a)); #else - EIGEN_ALIGN_MAX int64_t lvals[2]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) int64_t lvals[2]; pstore(lvals, a); - EIGEN_ALIGN_MAX float fvals[2] = {static_cast(lvals[0]), static_cast(lvals[1])}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) + float fvals[2] = {static_cast(lvals[0]), static_cast(lvals[1])}; return pload(fvals); #endif } @@ -1319,11 +1321,12 @@ EIGEN_STRONG_INLINE Packet4f pcast(const Packet2ul& a, cons #if EIGEN_ARCH_ARM64 return vcombine_f32(vcvt_f32_f64(vcvtq_f64_u64(a)), vcvt_f32_f64(vcvtq_f64_u64(b))); #else - EIGEN_ALIGN_MAX uint64_t uvals[4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) uint64_t uvals[4]; pstore(uvals, a); pstore(uvals + 2, b); - EIGEN_ALIGN_MAX float fvals[4] = {static_cast(uvals[0]), static_cast(uvals[1]), - static_cast(uvals[2]), static_cast(uvals[3])}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) + float fvals[4] = {static_cast(uvals[0]), static_cast(uvals[1]), static_cast(uvals[2]), + static_cast(uvals[3])}; return pload(fvals); #endif } @@ -1332,9 +1335,10 @@ EIGEN_STRONG_INLINE Packet2f pcast(const Packet2ul& a) { #if EIGEN_ARCH_ARM64 return vcvt_f32_f64(vcvtq_f64_u64(a)); #else - EIGEN_ALIGN_MAX uint64_t uvals[2]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) uint64_t uvals[2]; pstore(uvals, a); - EIGEN_ALIGN_MAX float fvals[2] = {static_cast(uvals[0]), static_cast(uvals[1])}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) + float fvals[2] = {static_cast(uvals[0]), static_cast(uvals[1])}; return pload(fvals); #endif } diff --git a/Eigen/src/LU/arch/InverseSize4.h b/Eigen/src/LU/arch/InverseSize4.h index 5d12f8cd9..0ee47749d 100644 --- a/Eigen/src/LU/arch/InverseSize4.h +++ b/Eigen/src/LU/arch/InverseSize4.h @@ -149,7 +149,8 @@ struct compute_inverse_size4(0x80000000u); - EIGEN_ALIGN_MAX const float sign_mask[4] = {0.0f, neg_zero, neg_zero, 0.0f}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) + const float sign_mask[4] = {0.0f, neg_zero, neg_zero, 0.0f}; const Packet4f p4f_sign_PNNP = pload(sign_mask); rd = pxor(rd, p4f_sign_PNNP); iA = pmul(iA, rd); @@ -326,8 +327,8 @@ struct compute_inverse_size4(0x8000000000000000ull); - EIGEN_ALIGN_MAX const double sign_mask1[2] = {0.0, neg_zero}; - EIGEN_ALIGN_MAX const double sign_mask2[2] = {neg_zero, 0.0}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) const double sign_mask1[2] = {0.0, neg_zero}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) const double sign_mask2[2] = {neg_zero, 0.0}; const Packet2d sign_PN = pload(sign_mask1); const Packet2d sign_NP = pload(sign_mask2); d1 = pxor(rd, sign_PN); diff --git a/test/geo_quaternion.cpp b/test/geo_quaternion.cpp index ca8f55106..3c45b01af 100644 --- a/test/geo_quaternion.cpp +++ b/test/geo_quaternion.cpp @@ -326,8 +326,8 @@ void mapQuaternion(void) { Vector3 v0 = Vector3::Random(), v1 = Vector3::Random(); Scalar a = internal::random(-Scalar(EIGEN_PI), Scalar(EIGEN_PI)); - EIGEN_ALIGN_MAX Scalar array1[4]; - EIGEN_ALIGN_MAX Scalar array2[4]; + EIGEN_ALIGN_TO_BOUNDARY(Aligned) Scalar array1[4]; + EIGEN_ALIGN_TO_BOUNDARY(Aligned) Scalar array2[4]; EIGEN_ALIGN_MAX Scalar array3[4 + 1]; Scalar* array3unaligned = array3 + 1; diff --git a/test/packetmath.cpp b/test/packetmath.cpp index 8a2370330..067ada0b4 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp @@ -12,6 +12,8 @@ #include "packetmath_test_shared.h" #include "random_without_cast_overflow.h" +using internal::unpacket_traits; + template ::IsInteger || !NumTraits::IsSigned, int> = 0> inline T REF_ADD(const T& a, const T& b) { return a + b; @@ -398,9 +400,9 @@ void packetmath_boolean_mask_ops() { using RealScalar = typename NumTraits::Real; const int PacketSize = internal::unpacket_traits::size; const int size = 2 * PacketSize; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[size]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[size]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar ref[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar ref[size]; for (int i = 0; i < size; ++i) { data1[i] = internal::random(); @@ -433,8 +435,8 @@ template void packetmath_boolean_mask_ops_real() { const int PacketSize = internal::unpacket_traits::size; const int size = 2 * PacketSize; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[size]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[size]; for (int i = 0; i < PacketSize; ++i) { data1[i] = internal::random(); @@ -469,8 +471,8 @@ struct packetmath_boolean_mask_ops_notcomplex_test< static void run() { const int PacketSize = internal::unpacket_traits::size; const int size = 2 * PacketSize; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[size]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[size]; for (int i = 0; i < PacketSize; ++i) { data1[i] = internal::random(); @@ -508,9 +510,9 @@ struct packetmath_minus_zero_add_test::size; const int size = 2 * PacketSize; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[size] = {}; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[size] = {}; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar ref[size] = {}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[size] = {}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[size] = {}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar ref[size] = {}; for (int i = 0; i < PacketSize; ++i) { data1[i] = Scalar(-0.0); @@ -533,13 +535,13 @@ struct packetmath_integer_predicates_test< Scalar, Packet, std::enable_if_t::IsInteger && !std::is_same::value>> { static void run() { const int PacketSize = internal::unpacket_traits::size; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data[PacketSize]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar res[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar res[PacketSize]; // "True" is Scalar(1) in the scalar mask convention and all-ones bits in the packet one; // ptrue of the tested Packet type yields the right one either way (the runner also // instantiates Packet = Scalar). const Scalar scalar_true = internal::ptrue(Scalar(0)); - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar lane_true[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar lane_true[PacketSize]; internal::pstore(lane_true, internal::ptrue(internal::pset1(Scalar(0)))); const Scalar values[] = {Scalar(0), Scalar(1), @@ -623,10 +625,10 @@ void packetmath() { constexpr int max_size = PacketSize > 4 ? PacketSize : 4; const int size = PacketSize * max_size; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[size]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[size]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data3[size]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar ref[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data3[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar ref[size]; RealScalar refvalue = RealScalar(0); eigen_optimization_barrier_test::run(); @@ -908,9 +910,9 @@ void packetmath_real() { const int PacketSize = internal::unpacket_traits::size; const int size = PacketSize * 4; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[PacketSize * 4] = {}; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[PacketSize * 4] = {}; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar ref[PacketSize * 4] = {}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[PacketSize * 4] = {}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[PacketSize * 4] = {}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar ref[PacketSize * 4] = {}; // Negate with -0. if (PacketTraits::HasNegate) { @@ -1365,9 +1367,9 @@ std::enable_if_t run_ieee_cases(const FunctorT& fun) { } constexpr int size = PacketSize * 2; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[size]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[size]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar ref[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar ref[size]; for (int i = 0; i < size; ++i) { data1[i] = data2[i] = ref[i] = Scalar(0); } @@ -1447,9 +1449,9 @@ void packetmath_notcomplex() { typedef internal::packet_traits PacketTraits; const int PacketSize = internal::unpacket_traits::size; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[PacketSize * 4]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[PacketSize * 4]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar ref[PacketSize * 4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[PacketSize * 4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[PacketSize * 4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar ref[PacketSize * 4]; Array::Map(data1, PacketSize * 4).setRandom(); @@ -1731,12 +1733,12 @@ void packetmath_complex() { const int PacketSize = internal::unpacket_traits::size; const int size = PacketSize * 4; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[PacketSize * 4]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[PacketSize * 4]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar ref[PacketSize * 4]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar pval[PacketSize * 4]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) RealScalar realdata[PacketSize * 4]; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) RealScalar realref[PacketSize * 4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[PacketSize * 4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[PacketSize * 4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar ref[PacketSize * 4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar pval[PacketSize * 4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) RealScalar realdata[PacketSize * 4]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) RealScalar realref[PacketSize * 4]; for (int i = 0; i < size; ++i) { data1[i] = internal::random() * Scalar(1e2); @@ -1893,7 +1895,7 @@ template void packetmath_scatter_gather() { typedef typename NumTraits::Real RealScalar; const int PacketSize = internal::unpacket_traits::size; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[PacketSize]; RealScalar refvalue = RealScalar(0); for (int i = 0; i < PacketSize; ++i) { data1[i] = internal::random(); diff --git a/unsupported/Eigen/src/AutoDiff/CoherentPadOp.h b/unsupported/Eigen/src/AutoDiff/CoherentPadOp.h index 59733c41a..e97378fda 100644 --- a/unsupported/Eigen/src/AutoDiff/CoherentPadOp.h +++ b/unsupported/Eigen/src/AutoDiff/CoherentPadOp.h @@ -128,7 +128,8 @@ struct unary_evaluator> return m_argImpl.template packet(index); } else if (index < m_size.value()) { // Partial packet. - EIGEN_ALIGN_MAX std::remove_const_t values[kPacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) + std::remove_const_t values[kPacketSize]; const int partial = m_size.value() - index; for (int i = 0; i < partial && i < kPacketSize; ++i) { values[i] = m_argImpl.coeff(index + i); diff --git a/unsupported/Eigen/src/Tensor/TensorBroadcasting.h b/unsupported/Eigen/src/Tensor/TensorBroadcasting.h index c6a80b031..cdde13389 100644 --- a/unsupported/Eigen/src/Tensor/TensorBroadcasting.h +++ b/unsupported/Eigen/src/Tensor/TensorBroadcasting.h @@ -346,7 +346,8 @@ struct TensorEvaluator, Device> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetOneByNByOne(Index index) const { eigen_assert(index + PacketSize - 1 < dimensions().TotalSize()); - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; Index startDim, endDim; Index inputIndex, outputOffset, batchedIndex; @@ -397,7 +398,8 @@ struct TensorEvaluator, Device> { if (inputIndex + PacketSize <= M) { return m_impl.template packet(inputIndex); } else { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { if (inputIndex > M - 1) { @@ -425,7 +427,8 @@ struct TensorEvaluator, Device> { if (outputOffset + PacketSize <= M) { return internal::pset1(m_impl.coeff(inputIndex)); } else { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { if (outputOffset < M) { @@ -483,7 +486,8 @@ struct TensorEvaluator, Device> { if (innermostLoc + PacketSize <= m_impl.dimensions()[0]) { return m_impl.template packet(inputIndex); } else { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; values[0] = m_impl.coeff(inputIndex); EIGEN_UNROLL_LOOP for (int i = 1; i < PacketSize; ++i) { @@ -539,7 +543,8 @@ struct TensorEvaluator, Device> { if (innermostLoc + PacketSize <= m_impl.dimensions()[NumDims - 1]) { return m_impl.template packet(inputIndex); } else { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; values[0] = m_impl.coeff(inputIndex); EIGEN_UNROLL_LOOP for (int i = 1; i < PacketSize; ++i) { diff --git a/unsupported/Eigen/src/Tensor/TensorChipping.h b/unsupported/Eigen/src/Tensor/TensorChipping.h index d3fab35e8..44325c473 100644 --- a/unsupported/Eigen/src/Tensor/TensorChipping.h +++ b/unsupported/Eigen/src/Tensor/TensorChipping.h @@ -224,7 +224,8 @@ struct TensorEvaluator, Device> { return m_impl.template packet(inputIndex); } else { // Cross the stride boundary. Fallback to slow path. - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = coeff(index); @@ -358,7 +359,8 @@ struct TensorEvaluator, Device> { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetInnerChipping(Index index) const { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; Index inputIndex = srcCoeffInnerChipping(index); EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { @@ -448,7 +450,8 @@ struct TensorEvaluator, Device> this->m_impl.template writePacket(inputIndex, x); } else { // Cross stride boundary. Fallback to slow path. - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; internal::pstore(values, x); EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { @@ -483,7 +486,8 @@ struct TensorEvaluator, Device> private: template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacketInnerChipping(Index index, const PacketReturnType& x) const { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; internal::pstore(values, x); Index inputIndex = this->srcCoeffInnerChipping(index); EIGEN_UNROLL_LOOP diff --git a/unsupported/Eigen/src/Tensor/TensorConcatenation.h b/unsupported/Eigen/src/Tensor/TensorConcatenation.h index 93f7bad19..10228d642 100644 --- a/unsupported/Eigen/src/Tensor/TensorConcatenation.h +++ b/unsupported/Eigen/src/Tensor/TensorConcatenation.h @@ -497,7 +497,7 @@ struct TensorEvaluator::alignment) CoeffReturnType values[packetSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < packetSize; ++i) { values[i] = coeff(index + i); @@ -597,7 +597,7 @@ struct TensorEvaluator, D EIGEN_STATIC_ASSERT((packetSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index + packetSize - 1 < this->dimensions().TotalSize()); - EIGEN_ALIGN_MAX CoeffReturnType values[packetSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) CoeffReturnType values[packetSize]; internal::pstore(values, x); for (int i = 0; i < packetSize; ++i) { coeffRef(index + i) = values[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorContractionMapper.h b/unsupported/Eigen/src/Tensor/TensorContractionMapper.h index 6ff5d8bc4..f819761c3 100644 --- a/unsupported/Eigen/src/Tensor/TensorContractionMapper.h +++ b/unsupported/Eigen/src/Tensor/TensorContractionMapper.h @@ -282,7 +282,7 @@ class BaseTensorContractionMapper } } - EIGEN_ALIGN_MAX Scalar data[packet_size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data[packet_size]; data[0] = this->m_tensor.coeff(first); EIGEN_UNROLL_LOOP @@ -301,7 +301,7 @@ class BaseTensorContractionMapper std::enable_if_t::size != packet_size, PacketT> load(Index i, Index j) const { const Index requested_packet_size = internal::unpacket_traits::size; - EIGEN_ALIGN_MAX Scalar data[requested_packet_size]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data[requested_packet_size]; const IndexPair indexPair = this->computeIndexPair(i, j, requested_packet_size - 1); const Index first = indexPair.first; @@ -341,13 +341,13 @@ class BaseTensorContractionMapper EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketT loadPacket(Index i, Index j) const { - EIGEN_ALIGN_MAX Scalar data[1]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data[1]; data[0] = this->m_tensor.coeff(this->computeIndex(i, j)); return pload(data); } template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketT load(Index i, Index j) const { - EIGEN_ALIGN_MAX Scalar data[1]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data[1]; data[0] = this->m_tensor.coeff(this->computeIndex(i, j)); return pload(data); } diff --git a/unsupported/Eigen/src/Tensor/TensorConversion.h b/unsupported/Eigen/src/Tensor/TensorConversion.h index 3ba153a91..82a279649 100644 --- a/unsupported/Eigen/src/Tensor/TensorConversion.h +++ b/unsupported/Eigen/src/Tensor/TensorConversion.h @@ -133,7 +133,8 @@ struct PacketConverter typedef typename internal::unpacket_traits::type SrcType; typedef typename internal::unpacket_traits::type TgtType; internal::scalar_cast_op converter; - EIGEN_ALIGN_MAX typename internal::unpacket_traits::type values[TgtPacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + typename internal::unpacket_traits::type values[TgtPacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < TgtPacketSize; ++i) { values[i] = converter(m_impl.coeff(index + i)); @@ -234,7 +235,8 @@ struct PacketConv { static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TargetPacket run(const TensorEvaluator& impl, Index index) { internal::scalar_cast_op converter; - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = converter(impl.coeff(index + i)); @@ -268,7 +270,8 @@ struct PacketConv static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TargetPacket run(const TensorEvaluator& impl, Index index) { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; for (int i = 0; i < PacketSize; ++i) values[i] = impl.coeff(index + i); return internal::pload(values); } diff --git a/unsupported/Eigen/src/Tensor/TensorConvolution.h b/unsupported/Eigen/src/Tensor/TensorConvolution.h index 30a8c42b7..70357aa86 100644 --- a/unsupported/Eigen/src/Tensor/TensorConvolution.h +++ b/unsupported/Eigen/src/Tensor/TensorConvolution.h @@ -415,7 +415,7 @@ struct TensorEvaluator::alignment) Scalar data[PacketSize]; data[0] = Scalar(0); convolve(startInputs[0], 0, NumKernelDims - 1, data[0]); for (int i = 1; i < PacketSize - 1; ++i) { diff --git a/unsupported/Eigen/src/Tensor/TensorEvaluator.h b/unsupported/Eigen/src/Tensor/TensorEvaluator.h index be53d75a5..b0aec5413 100644 --- a/unsupported/Eigen/src/Tensor/TensorEvaluator.h +++ b/unsupported/Eigen/src/Tensor/TensorEvaluator.h @@ -1047,7 +1047,8 @@ struct TensorEvaluator template = true> EIGEN_DEVICE_FUNC PacketReturnType packet(Index index) const { - EIGEN_ALIGN_TO_BOUNDARY(sizeof(PacketReturnType)) std::remove_const_t arr[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t arr[PacketSize]; EIGEN_UNROLL_LOOP for (Index i = 0; i < PacketSize; ++i) { arr[i] = m_condImpl.coeff(index + i) ? Scalar(-1) : Scalar(0); diff --git a/unsupported/Eigen/src/Tensor/TensorFixedSize.h b/unsupported/Eigen/src/Tensor/TensorFixedSize.h index 2c82ab309..6ddba03f3 100644 --- a/unsupported/Eigen/src/Tensor/TensorFixedSize.h +++ b/unsupported/Eigen/src/Tensor/TensorFixedSize.h @@ -42,7 +42,7 @@ class TensorFixedSize : public TensorBase 0), + IsAligned = bool(EIGEN_MAX_STATIC_ALIGN_BYTES >= Aligned), PacketAccess = (internal::packet_traits::size > 1), BlockAccess = false, PreferBlockAccess = false, diff --git a/unsupported/Eigen/src/Tensor/TensorGenerator.h b/unsupported/Eigen/src/Tensor/TensorGenerator.h index 0394df244..392828035 100644 --- a/unsupported/Eigen/src/Tensor/TensorGenerator.h +++ b/unsupported/Eigen/src/Tensor/TensorGenerator.h @@ -131,7 +131,8 @@ struct TensorEvaluator, Device> { const int packetSize = PacketType::size; eigen_assert(index + packetSize - 1 < dimensions().TotalSize()); - EIGEN_ALIGN_MAX std::remove_const_t values[packetSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[packetSize]; for (int i = 0; i < packetSize; ++i) { values[i] = coeff(index + i); } diff --git a/unsupported/Eigen/src/Tensor/TensorImagePatch.h b/unsupported/Eigen/src/Tensor/TensorImagePatch.h index bb88960da..b7e7ffe2e 100644 --- a/unsupported/Eigen/src/Tensor/TensorImagePatch.h +++ b/unsupported/Eigen/src/Tensor/TensorImagePatch.h @@ -549,7 +549,8 @@ struct TensorEvaluator, Device> { protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetWithPossibleZero(Index index) const { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = coeff(index + i); diff --git a/unsupported/Eigen/src/Tensor/TensorInflation.h b/unsupported/Eigen/src/Tensor/TensorInflation.h index 66e980913..344a7e172 100644 --- a/unsupported/Eigen/src/Tensor/TensorInflation.h +++ b/unsupported/Eigen/src/Tensor/TensorInflation.h @@ -196,7 +196,8 @@ struct TensorEvaluator, Device> { EIGEN_STATIC_ASSERT((PacketSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE) eigen_assert(index + PacketSize - 1 < dimensions().TotalSize()); - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = coeff(index + i); diff --git a/unsupported/Eigen/src/Tensor/TensorMorphing.h b/unsupported/Eigen/src/Tensor/TensorMorphing.h index 888a0af08..9543a6830 100644 --- a/unsupported/Eigen/src/Tensor/TensorMorphing.h +++ b/unsupported/Eigen/src/Tensor/TensorMorphing.h @@ -511,7 +511,8 @@ struct TensorEvaluator, Devi PacketReturnType rslt = m_impl.template packet(inputIndices[0]); return rslt; } else { - EIGEN_ALIGN_MAX std::remove_const_t values[packetSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[packetSize]; values[0] = m_impl.coeff(inputIndices[0]); values[packetSize - 1] = m_impl.coeff(inputIndices[1]); EIGEN_UNROLL_LOOP @@ -697,7 +698,8 @@ struct TensorEvaluator, Device> if (inputIndices[1] - inputIndices[0] == packetSize - 1) { this->m_impl.template writePacket(inputIndices[0], x); } else { - EIGEN_ALIGN_MAX CoeffReturnType values[packetSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + CoeffReturnType values[packetSize]; internal::pstore(values, x); this->m_impl.coeffRef(inputIndices[0]) = values[0]; this->m_impl.coeffRef(inputIndices[1]) = values[packetSize - 1]; @@ -902,7 +904,8 @@ struct TensorEvaluator values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; if (inner_pos + PacketSize <= inner_size) { const Index inner_stride = m_inputStrides[inner_dim]; if (inner_stride == 1) { @@ -1053,7 +1056,8 @@ struct TensorEvaluatorm_dimensions[inner_dim]; Index inner_pos; const Index base = this->srcCoeffInner(index, inner_pos); - EIGEN_ALIGN_TO_BOUNDARY(sizeof(PacketReturnType)) CoeffReturnType values[Base::PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + CoeffReturnType values[Base::PacketSize]; if (inner_pos + Base::PacketSize <= inner_size) { const Index inner_stride = this->m_inputStrides[inner_dim]; if (inner_stride == 1) { diff --git a/unsupported/Eigen/src/Tensor/TensorPadding.h b/unsupported/Eigen/src/Tensor/TensorPadding.h index de7e87fb2..d9fface33 100644 --- a/unsupported/Eigen/src/Tensor/TensorPadding.h +++ b/unsupported/Eigen/src/Tensor/TensorPadding.h @@ -605,7 +605,8 @@ struct TensorEvaluator, Device } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetWithPossibleZero(Index index) const { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = coeff(index + i); diff --git a/unsupported/Eigen/src/Tensor/TensorPatch.h b/unsupported/Eigen/src/Tensor/TensorPatch.h index 5bca06fc2..d8f787540 100644 --- a/unsupported/Eigen/src/Tensor/TensorPatch.h +++ b/unsupported/Eigen/src/Tensor/TensorPatch.h @@ -218,7 +218,8 @@ struct TensorEvaluator, Device> { PacketReturnType rslt = m_impl.template packet(inputIndices[0]); return rslt; } else { - EIGEN_ALIGN_MAX CoeffReturnType values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + CoeffReturnType values[PacketSize]; values[0] = m_impl.coeff(inputIndices[0]); values[PacketSize - 1] = m_impl.coeff(inputIndices[1]); EIGEN_UNROLL_LOOP diff --git a/unsupported/Eigen/src/Tensor/TensorRandom.h b/unsupported/Eigen/src/Tensor/TensorRandom.h index 365ef0c2d..173c93dd9 100644 --- a/unsupported/Eigen/src/Tensor/TensorRandom.h +++ b/unsupported/Eigen/src/Tensor/TensorRandom.h @@ -161,7 +161,7 @@ class UniformRandomGenerator { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(Index i) const { const int packetSize = internal::unpacket_traits::size; - EIGEN_ALIGN_MAX T values[packetSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) T values[packetSize]; #ifdef EIGEN_USE_SYCL if (!m_exec_once) { // This is the second stage of adding thread Id to the CPU clock seed and build unique seed per thread @@ -275,7 +275,7 @@ class NormalRandomGenerator { template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(Index i) const { const int packetSize = internal::unpacket_traits::size; - EIGEN_ALIGN_MAX T values[packetSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) T values[packetSize]; #ifdef EIGEN_USE_SYCL if (!m_exec_once) { // This is the second stage of adding thread Id to the CPU clock seed and build unique seed per thread diff --git a/unsupported/Eigen/src/Tensor/TensorReduction.h b/unsupported/Eigen/src/Tensor/TensorReduction.h index e48d53324..1edaf8f72 100644 --- a/unsupported/Eigen/src/Tensor/TensorReduction.h +++ b/unsupported/Eigen/src/Tensor/TensorReduction.h @@ -905,7 +905,8 @@ struct TensorReductionEvaluatorBase values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_IF_CONSTEXPR (ReducingInnerMostDims) { const Index num_values_to_reduce = (static_cast(Layout) == static_cast(ColMajor)) ? m_preservedStrides[0] diff --git a/unsupported/Eigen/src/Tensor/TensorReverse.h b/unsupported/Eigen/src/Tensor/TensorReverse.h index b1f479eba..291f555f2 100644 --- a/unsupported/Eigen/src/Tensor/TensorReverse.h +++ b/unsupported/Eigen/src/Tensor/TensorReverse.h @@ -200,7 +200,8 @@ struct TensorEvaluator, Device // Slow path: the packet crosses an inner-slice boundary, so the // contiguous-load trick does not apply. - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = coeff(index + i); @@ -417,7 +418,7 @@ struct TensorEvaluator, Device> } // Slow path: the packet crosses an inner-slice boundary. - EIGEN_ALIGN_MAX CoeffReturnType values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) CoeffReturnType values[PacketSize]; internal::pstore(values, x); EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { diff --git a/unsupported/Eigen/src/Tensor/TensorRoll.h b/unsupported/Eigen/src/Tensor/TensorRoll.h index 5b5cea7cc..c11fe2aa0 100644 --- a/unsupported/Eigen/src/Tensor/TensorRoll.h +++ b/unsupported/Eigen/src/Tensor/TensorRoll.h @@ -199,7 +199,8 @@ struct TensorEvaluator, Device> { } // Slow path: the packet straddles a slice boundary on either side. - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = coeff(index + i); @@ -356,7 +357,7 @@ struct TensorEvaluator, Device> template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) const { eigen_assert(index + PacketSize - 1 < dimensions().TotalSize()); - EIGEN_ALIGN_MAX CoeffReturnType values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) CoeffReturnType values[PacketSize]; internal::pstore(values, x); EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { diff --git a/unsupported/Eigen/src/Tensor/TensorShuffling.h b/unsupported/Eigen/src/Tensor/TensorShuffling.h index 2dea9f4ff..8c203e749 100644 --- a/unsupported/Eigen/src/Tensor/TensorShuffling.h +++ b/unsupported/Eigen/src/Tensor/TensorShuffling.h @@ -171,7 +171,8 @@ struct TensorEvaluator, Device> { struct InnerRunLoader { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType Run(const Self& self, Index base, Index inner_stride) { - EIGEN_ALIGN_TO_BOUNDARY(sizeof(PacketReturnType)) std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = self.m_impl.coeff(base + i * inner_stride); @@ -188,7 +189,8 @@ struct TensorEvaluator, Device> { // Inner dimension not shuffled: one contiguous load. return self.m_impl.template packet(base); } - EIGEN_ALIGN_TO_BOUNDARY(sizeof(PacketReturnType)) std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = self.m_impl.coeff(base + i * inner_stride); @@ -207,7 +209,8 @@ struct TensorEvaluator, Device> { } // The packet crosses an inner-run boundary: assemble it scalar by scalar. - EIGEN_ALIGN_TO_BOUNDARY(sizeof(PacketReturnType)) std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = self.coeff(index + i); @@ -219,7 +222,8 @@ struct TensorEvaluator, Device> { struct PacketLoader { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType Run(const Self& self, Index index) { if (self.m_is_identity) { - EIGEN_ALIGN_TO_BOUNDARY(sizeof(PacketReturnType)) std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = self.m_impl.coeff(index + i); @@ -432,7 +436,8 @@ struct TensorEvaluator, Device> if (inner_stride == 1 && InnerRunWriter::Run(*this, base, x)) { return; } - EIGEN_ALIGN_TO_BOUNDARY(sizeof(PacketReturnType)) std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; internal::pstore(values, x); EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { @@ -442,7 +447,8 @@ struct TensorEvaluator, Device> } // The packet crosses an inner-run boundary: scatter scalar by scalar. - EIGEN_ALIGN_TO_BOUNDARY(sizeof(PacketReturnType)) std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; internal::pstore(values, x); EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { diff --git a/unsupported/Eigen/src/Tensor/TensorStriding.h b/unsupported/Eigen/src/Tensor/TensorStriding.h index 665289bd5..26a6b6872 100644 --- a/unsupported/Eigen/src/Tensor/TensorStriding.h +++ b/unsupported/Eigen/src/Tensor/TensorStriding.h @@ -169,7 +169,8 @@ struct TensorEvaluator, Device> { PacketReturnType rslt = m_impl.template packet(inputIndices[0]); return rslt; } else { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; values[0] = m_impl.coeff(inputIndices[0]); values[PacketSize - 1] = m_impl.coeff(inputIndices[1]); EIGEN_UNROLL_LOOP @@ -289,7 +290,7 @@ struct TensorEvaluator, Device> if (inputIndices[1] - inputIndices[0] == PacketSize - 1) { this->m_impl.template writePacket(inputIndices[0], x); } else { - EIGEN_ALIGN_MAX Scalar values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) Scalar values[PacketSize]; internal::pstore(values, x); this->m_impl.coeffRef(inputIndices[0]) = values[0]; this->m_impl.coeffRef(inputIndices[1]) = values[PacketSize - 1]; diff --git a/unsupported/Eigen/src/Tensor/TensorVolumePatch.h b/unsupported/Eigen/src/Tensor/TensorVolumePatch.h index 59b661632..f8796480b 100644 --- a/unsupported/Eigen/src/Tensor/TensorVolumePatch.h +++ b/unsupported/Eigen/src/Tensor/TensorVolumePatch.h @@ -532,7 +532,8 @@ struct TensorEvaluator, D protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetWithPossibleZero(Index index) const { - EIGEN_ALIGN_MAX std::remove_const_t values[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::alignment) + std::remove_const_t values[PacketSize]; EIGEN_UNROLL_LOOP for (int i = 0; i < PacketSize; ++i) { values[i] = coeff(index + i); diff --git a/unsupported/test/special_packetmath.cpp b/unsupported/test/special_packetmath.cpp index 5fa29cdcb..83d7c23a7 100644 --- a/unsupported/test/special_packetmath.cpp +++ b/unsupported/test/special_packetmath.cpp @@ -13,6 +13,8 @@ #include "packetmath_test_shared.h" #include "../Eigen/SpecialFunctions" +using internal::unpacket_traits; + #if EIGEN_ARCH_ARM // Note: 32-bit arm always flushes subnormals to zero. #define MAYBE_FLUSH(op) \ @@ -34,9 +36,9 @@ void packetmath_real() { const int PacketSize = internal::unpacket_traits::size; const int size = PacketSize * 4; - EIGEN_ALIGN_MAX Scalar data1[PacketSize * 4] = {}; - EIGEN_ALIGN_MAX Scalar data2[PacketSize * 4] = {}; - EIGEN_ALIGN_MAX Scalar ref[PacketSize * 4] = {}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data1[PacketSize * 4] = {}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar data2[PacketSize * 4] = {}; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits::alignment) Scalar ref[PacketSize * 4] = {}; { data1[0] = std::numeric_limits::quiet_NaN(); diff --git a/unsupported/test/tensor_fixed_size.cpp b/unsupported/test/tensor_fixed_size.cpp index bbf31466f..029044784 100644 --- a/unsupported/test/tensor_fixed_size.cpp +++ b/unsupported/test/tensor_fixed_size.cpp @@ -254,6 +254,41 @@ static void test_array() { } } +// Sanity check: `TensorFixedSize` whose element count isn't a multiple of the packet width +// (5 floats) must still be safely usable when embedded in a larger aligned struct at +// a non-16-byte-aligned offset. Since `IsAligned` (and thus the `Aligned`/`Unaligned` +// `LoadMode` `TensorEvaluator` picks for packet ops) is a flat, `Size`-independent trait, +// it must not assume a stronger alignment than the storage actually provides for odd +// sizes like this one. +struct TensorFixedSizeEmbeddingHolder { + float pad; + TensorFixedSize > tensor; +}; + +static void test_embedded_odd_size() { + EIGEN_ALIGN_MAX TensorFixedSizeEmbeddingHolder lhs; + EIGEN_ALIGN_MAX TensorFixedSizeEmbeddingHolder rhs; + + for (int i = 0; i < 5; ++i) { + rhs.tensor(i) = static_cast(i + 1); + } + + lhs.tensor = rhs.tensor + rhs.tensor; + + for (int i = 0; i < 5; ++i) { + VERIFY_IS_APPROX(lhs.tensor(i), 2.0f * static_cast(i + 1)); + } +} + +// Regression test: `IsAligned` is consumed by evaluators as the legacy `Aligned` load mode, +// which promises `Aligned16`-byte alignment regardless of what `EIGEN_MAX_STATIC_ALIGN_BYTES` +// is actually set to. A build configured with `0 < EIGEN_MAX_STATIC_ALIGN_BYTES < Aligned` +// (e.g. 8) still only 8-byte-aligns `TensorStorage`'s fixed array via `EIGEN_ALIGN_MAX`, so +// `IsAligned` must not claim aligned access is safe unless the configured static alignment +// actually reaches `Aligned` (16) bytes. +static_assert(TensorFixedSize >::IsAligned == (EIGEN_MAX_STATIC_ALIGN_BYTES >= 16), + "IsAligned must match whether EIGEN_MAX_STATIC_ALIGN_BYTES actually provides Aligned16 storage"); + EIGEN_DECLARE_TEST(tensor_fixed_size) { CALL_SUBTEST(test_0d()); CALL_SUBTEST(test_1d()); @@ -261,4 +296,5 @@ EIGEN_DECLARE_TEST(tensor_fixed_size) { CALL_SUBTEST(test_2d()); CALL_SUBTEST(test_3d()); CALL_SUBTEST(test_array()); + CALL_SUBTEST(test_embedded_odd_size()); } diff --git a/unsupported/test/tensor_random.cpp b/unsupported/test/tensor_random.cpp index e1fca6dc8..61ee893eb 100644 --- a/unsupported/test/tensor_random.cpp +++ b/unsupported/test/tensor_random.cpp @@ -97,7 +97,8 @@ struct MyGenerator { internal::packet_traits::type packetOp(Eigen::DenseIndex packet_location, Eigen::DenseIndex /*unused*/ = 0) const { const int packetSize = internal::packet_traits::size; - EIGEN_ALIGN_MAX int values[packetSize]; + EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits::type>::alignment) + int values[packetSize]; for (int i = 0; i < packetSize; ++i) { values[i] = static_cast(3 * (packet_location + i)); }