clang-format: register EIGEN_IF_CONSTEXPR as an IfMacro

libeigen/eigen!2604

Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
This commit is contained in:
Rasmus Munk Larsen
2026-06-06 13:08:44 -07:00
co-authored by Rasmus Munk Larsen
parent 7b56b05698
commit 7966ea495e
97 changed files with 1714 additions and 1751 deletions
+2
View File
@@ -9,6 +9,8 @@ StatementMacros:
- EIGEN_STATIC_ASSERT
- EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED
- EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN
IfMacros:
- EIGEN_IF_CONSTEXPR
SortIncludes: false
AttributeMacros:
- EIGEN_STRONG_INLINE
@@ -176,35 +176,28 @@ class AccelerateImpl : public SparseSolverBase<AccelerateImpl<MatrixType_, UpLo_
AccelerateImpl() {
m_isInitialized = false;
EIGEN_IF_CONSTEXPR((UpLo_ & Symmetric) == Symmetric) {
EIGEN_IF_CONSTEXPR ((UpLo_ & Symmetric) == Symmetric) {
m_sparseKind = SparseSymmetric;
m_triType = (UpLo_ & Lower) ? SparseLowerTriangle : SparseUpperTriangle;
}
else EIGEN_IF_CONSTEXPR((UpLo_ & UnitLower) == UnitLower) {
} else EIGEN_IF_CONSTEXPR ((UpLo_ & UnitLower) == UnitLower) {
m_sparseKind = SparseUnitTriangular;
m_triType = SparseLowerTriangle;
}
else EIGEN_IF_CONSTEXPR((UpLo_ & UnitUpper) == UnitUpper) {
} else EIGEN_IF_CONSTEXPR ((UpLo_ & UnitUpper) == UnitUpper) {
m_sparseKind = SparseUnitTriangular;
m_triType = SparseUpperTriangle;
}
else EIGEN_IF_CONSTEXPR((UpLo_ & StrictlyLower) == StrictlyLower) {
} else EIGEN_IF_CONSTEXPR ((UpLo_ & StrictlyLower) == StrictlyLower) {
m_sparseKind = SparseTriangular;
m_triType = SparseLowerTriangle;
}
else EIGEN_IF_CONSTEXPR((UpLo_ & StrictlyUpper) == StrictlyUpper) {
} else EIGEN_IF_CONSTEXPR ((UpLo_ & StrictlyUpper) == StrictlyUpper) {
m_sparseKind = SparseTriangular;
m_triType = SparseUpperTriangle;
}
else EIGEN_IF_CONSTEXPR((UpLo_ & Lower) == Lower) {
} else EIGEN_IF_CONSTEXPR ((UpLo_ & Lower) == Lower) {
m_sparseKind = SparseTriangular;
m_triType = SparseLowerTriangle;
}
else EIGEN_IF_CONSTEXPR((UpLo_ & Upper) == Upper) {
} else EIGEN_IF_CONSTEXPR ((UpLo_ & Upper) == Upper) {
m_sparseKind = SparseTriangular;
m_triType = SparseUpperTriangle;
}
else {
} else {
m_sparseKind = SparseOrdinary;
m_triType = (UpLo_ & Lower) ? SparseLowerTriangle : SparseUpperTriangle;
}
+2 -2
View File
@@ -332,8 +332,8 @@ struct ldlt_inplace<Lower> {
mat.coeffRef(i, k) = numext::conj(mat.coeffRef(index_of_biggest_in_corner, i));
mat.coeffRef(index_of_biggest_in_corner, i) = numext::conj(tmp);
}
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex)
mat.coeffRef(index_of_biggest_in_corner, k) = numext::conj(mat.coeff(index_of_biggest_in_corner, k));
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex)
mat.coeffRef(index_of_biggest_in_corner, k) = numext::conj(mat.coeff(index_of_biggest_in_corner, k));
}
// partition the matrix:
+4 -4
View File
@@ -83,11 +83,11 @@ cholmod_sparse viewAsCholmod(Ref<SparseMatrix<Scalar_, Options_, StorageIndex_>
res.dtype = 0;
res.stype = -1;
EIGEN_IF_CONSTEXPR((std::is_same<StorageIndex_, int>::value)) { res.itype = CHOLMOD_INT; }
else EIGEN_IF_CONSTEXPR((std::is_same<StorageIndex_, SuiteSparse_long>::value)) {
EIGEN_IF_CONSTEXPR ((std::is_same<StorageIndex_, int>::value)) {
res.itype = CHOLMOD_INT;
} else EIGEN_IF_CONSTEXPR ((std::is_same<StorageIndex_, SuiteSparse_long>::value)) {
res.itype = CHOLMOD_LONG;
}
else {
} else {
eigen_assert(false && "Index type not supported yet");
}
+12 -16
View File
@@ -115,10 +115,9 @@ class Concat : public internal::dense_xpr_base<Concat<Direction, LhsType, RhsTyp
int(RhsType::ColsAtCompileTime) == Dynamic ||
int(LhsType::ColsAtCompileTime) == int(RhsType::ColsAtCompileTime),
YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES)
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
eigen_assert(lhs.cols() == rhs.cols() && "vcat: number of columns must match");
}
else {
} else {
eigen_assert(lhs.rows() == rhs.rows() && "hcat: number of rows must match");
}
}
@@ -175,13 +174,12 @@ struct evaluator<Concat<Direction, LhsType, RhsType>> : evaluator_base<Concat<Di
m_lhsCols(xpr.lhs().cols()) {}
EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
if (row < m_lhsRows.value())
return m_lhsImpl.coeff(row, col);
else
return m_rhsImpl.coeff(row - m_lhsRows.value(), col);
}
else {
} else {
if (col < m_lhsCols.value())
return m_lhsImpl.coeff(row, col);
else
@@ -200,22 +198,21 @@ struct evaluator<Concat<Direction, LhsType, RhsType>> : evaluator_base<Concat<Di
template <int LoadMode, typename PacketType>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
constexpr int packetSize = unpacket_traits<PacketType>::size;
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
const Index boundary = m_lhsRows.value();
if (row >= boundary) return m_rhsImpl.template packet<LoadMode, PacketType>(row - boundary, col);
// Column-major: inner=rows, packet extends along rows and may straddle the row boundary.
// Row-major: inner=cols, packet extends along cols — never crosses the row boundary.
EIGEN_IF_CONSTEXPR(!IsRowMajor) {
EIGEN_IF_CONSTEXPR (!IsRowMajor) {
if (row + packetSize > boundary) return packetBoundary<LoadMode, PacketType>(row, col);
}
return m_lhsImpl.template packet<LoadMode, PacketType>(row, col);
}
else {
} else {
const Index boundary = m_lhsCols.value();
if (col >= boundary) return m_rhsImpl.template packet<LoadMode, PacketType>(row, col - boundary);
// Row-major: inner=cols, packet extends along cols and may straddle the col boundary.
// Column-major: inner=rows, packet extends along rows — never crosses the col boundary.
EIGEN_IF_CONSTEXPR(IsRowMajor) {
EIGEN_IF_CONSTEXPR (IsRowMajor) {
if (col + packetSize > boundary) return packetBoundary<LoadMode, PacketType>(row, col);
}
return m_lhsImpl.template packet<LoadMode, PacketType>(row, col);
@@ -233,20 +230,19 @@ struct evaluator<Concat<Direction, LhsType, RhsType>> : evaluator_base<Concat<Di
template <int LoadMode, typename PacketType>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index row, Index col, Index begin, Index count) const {
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
const Index boundary = m_lhsRows.value();
if (row >= boundary)
return m_rhsImpl.template packetSegment<LoadMode, PacketType>(row - boundary, col, begin, count);
EIGEN_IF_CONSTEXPR(!IsRowMajor) {
EIGEN_IF_CONSTEXPR (!IsRowMajor) {
if (row + begin + count > boundary) return packetSegmentBoundary<LoadMode, PacketType>(row, col, begin, count);
}
return m_lhsImpl.template packetSegment<LoadMode, PacketType>(row, col, begin, count);
}
else {
} else {
const Index boundary = m_lhsCols.value();
if (col >= boundary)
return m_rhsImpl.template packetSegment<LoadMode, PacketType>(row, col - boundary, begin, count);
EIGEN_IF_CONSTEXPR(IsRowMajor) {
EIGEN_IF_CONSTEXPR (IsRowMajor) {
if (col + begin + count > boundary) return packetSegmentBoundary<LoadMode, PacketType>(row, col, begin, count);
}
return m_lhsImpl.template packetSegment<LoadMode, PacketType>(row, col, begin, count);
+20 -16
View File
@@ -1429,9 +1429,10 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
template <int LoadMode, typename PacketType>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const {
EIGEN_IF_CONSTEXPR(ForwardLinearAccess)
return m_argImpl.template packet<LoadMode, PacketType>(m_linear_offset.value() + index);
else return packet<LoadMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0);
EIGEN_IF_CONSTEXPR (ForwardLinearAccess)
return m_argImpl.template packet<LoadMode, PacketType>(m_linear_offset.value() + index);
else
return packet<LoadMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0);
}
template <int StoreMode, typename PacketType>
@@ -1441,10 +1442,11 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
template <int StoreMode, typename PacketType>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) {
EIGEN_IF_CONSTEXPR(ForwardLinearAccess)
return m_argImpl.template writePacket<StoreMode, PacketType>(m_linear_offset.value() + index, x);
else return writePacket<StoreMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index,
RowsAtCompileTime == 1 ? index : 0, x);
EIGEN_IF_CONSTEXPR (ForwardLinearAccess)
return m_argImpl.template writePacket<StoreMode, PacketType>(m_linear_offset.value() + index, x);
else
return writePacket<StoreMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0,
x);
}
template <int LoadMode, typename PacketType>
@@ -1455,10 +1457,11 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
template <int LoadMode, typename PacketType>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index index, Index begin, Index count) const {
EIGEN_IF_CONSTEXPR(ForwardLinearAccess)
return m_argImpl.template packetSegment<LoadMode, PacketType>(m_linear_offset.value() + index, begin, count);
else return packetSegment<LoadMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index,
RowsAtCompileTime == 1 ? index : 0, begin, count);
EIGEN_IF_CONSTEXPR (ForwardLinearAccess)
return m_argImpl.template packetSegment<LoadMode, PacketType>(m_linear_offset.value() + index, begin, count);
else
return packetSegment<LoadMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0,
begin, count);
}
template <int StoreMode, typename PacketType>
@@ -1471,11 +1474,12 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
template <int StoreMode, typename PacketType>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacketSegment(Index index, const PacketType& x, Index begin,
Index count) {
EIGEN_IF_CONSTEXPR(ForwardLinearAccess)
return m_argImpl.template writePacketSegment<StoreMode, PacketType>(m_linear_offset.value() + index, x, begin,
count);
else return writePacketSegment<StoreMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index,
RowsAtCompileTime == 1 ? index : 0, x, begin, count);
EIGEN_IF_CONSTEXPR (ForwardLinearAccess)
return m_argImpl.template writePacketSegment<StoreMode, PacketType>(m_linear_offset.value() + index, x, begin,
count);
else
return writePacketSegment<StoreMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index,
RowsAtCompileTime == 1 ? index : 0, x, begin, count);
}
protected:
+3 -2
View File
@@ -142,8 +142,9 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
#endif
DenseBase<Derived>::NullaryExpr(Index size, const CustomNullaryOp& func) {
EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { return CwiseNullaryOp<CustomNullaryOp, PlainObject>(1, size, func); }
else {
EIGEN_IF_CONSTEXPR (RowsAtCompileTime == 1) {
return CwiseNullaryOp<CustomNullaryOp, PlainObject>(1, size, func);
} else {
return CwiseNullaryOp<CustomNullaryOp, PlainObject>(size, 1, func);
}
}
+10 -14
View File
@@ -122,30 +122,26 @@ class CwiseTernaryOp : public CwiseTernaryOpImpl<TernaryOp, Arg1Type, Arg2Type,
EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Index rows() const {
// return the fixed size type if available to enable compile time
// optimizations
EIGEN_IF_CONSTEXPR(internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
internal::traits<internal::remove_all_t<Arg2Nested>>::RowsAtCompileTime == Dynamic) {
EIGEN_IF_CONSTEXPR (internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
internal::traits<internal::remove_all_t<Arg2Nested>>::RowsAtCompileTime == Dynamic) {
return m_arg3.rows();
}
else EIGEN_IF_CONSTEXPR(internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
internal::traits<internal::remove_all_t<Arg3Nested>>::RowsAtCompileTime == Dynamic) {
} else EIGEN_IF_CONSTEXPR (internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
internal::traits<internal::remove_all_t<Arg3Nested>>::RowsAtCompileTime == Dynamic) {
return m_arg2.rows();
}
else {
} else {
return m_arg1.rows();
}
}
EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Index cols() const {
// return the fixed size type if available to enable compile time
// optimizations
EIGEN_IF_CONSTEXPR(internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
internal::traits<internal::remove_all_t<Arg2Nested>>::ColsAtCompileTime == Dynamic) {
EIGEN_IF_CONSTEXPR (internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
internal::traits<internal::remove_all_t<Arg2Nested>>::ColsAtCompileTime == Dynamic) {
return m_arg3.cols();
}
else EIGEN_IF_CONSTEXPR(internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
internal::traits<internal::remove_all_t<Arg3Nested>>::ColsAtCompileTime == Dynamic) {
} else EIGEN_IF_CONSTEXPR (internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
internal::traits<internal::remove_all_t<Arg3Nested>>::ColsAtCompileTime == Dynamic) {
return m_arg2.cols();
}
else {
} else {
return m_arg1.cols();
}
}
+18 -12
View File
@@ -1448,8 +1448,9 @@ inline void pstore1(typename unpacket_traits<Packet>::type* to, const typename u
* The pointer \a from must be aligned on a \a Alignment bytes boundary. */
template <typename Packet, int Alignment>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_traits<Packet>::type* from) {
EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits<Packet>::alignment) { return pload<Packet>(from); }
else {
EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
return pload<Packet>(from);
} else {
return ploadu<Packet>(from);
}
}
@@ -1459,8 +1460,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_trai
template <typename Packet, int Alignment>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(const typename unpacket_traits<Packet>::type* from,
const Index n, const Index offset = 0) {
EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits<Packet>::alignment) { return pload_partial<Packet>(from, n, offset); }
else {
EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
return pload_partial<Packet>(from, n, offset);
} else {
return ploadu_partial<Packet>(from, n, offset);
}
}
@@ -1469,8 +1471,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(const typename unpac
* The pointer \a from must be aligned on a \a Alignment bytes boundary. */
template <typename Scalar, typename Packet, int Alignment>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& from) {
EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits<Packet>::alignment) { pstore(to, from); }
else {
EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
pstore(to, from);
} else {
pstoreu(to, from);
}
}
@@ -1480,8 +1483,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& fro
template <typename Scalar, typename Packet, int Alignment>
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret_partial(Scalar* to, const Packet& from, const Index n,
const Index offset = 0) {
EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits<Packet>::alignment) { pstore_partial(to, from, n, offset); }
else {
EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
pstore_partial(to, from, n, offset);
} else {
pstoreu_partial(to, from, n, offset);
}
}
@@ -1677,8 +1681,9 @@ template <typename Packet, int Alignment>
EIGEN_DEVICE_FUNC inline Packet ploadtSegment(const typename unpacket_traits<Packet>::type* from, Index begin,
Index count) {
constexpr int RequiredAlignment = unpacket_traits<Packet>::alignment;
EIGEN_IF_CONSTEXPR(Alignment >= RequiredAlignment) { return ploadSegment<Packet>(from, begin, count); }
else {
EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) {
return ploadSegment<Packet>(from, begin, count);
} else {
return ploaduSegment<Packet>(from, begin, count);
}
}
@@ -1688,8 +1693,9 @@ Elements outside of the range [begin, begin + count) are not defined.*/
template <typename Scalar, typename Packet, int Alignment>
EIGEN_DEVICE_FUNC inline void pstoretSegment(Scalar* to, const Packet& from, Index begin, Index count) {
constexpr int RequiredAlignment = unpacket_traits<Packet>::alignment;
EIGEN_IF_CONSTEXPR(Alignment >= RequiredAlignment) { pstoreSegment<Scalar, Packet>(to, from, begin, count); }
else {
EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) {
pstoreSegment<Scalar, Packet>(to, from, begin, count);
} else {
pstoreuSegment<Scalar, Packet>(to, from, begin, count);
}
}
+3 -2
View File
@@ -150,8 +150,9 @@ std::ostream& print_matrix(std::ostream& s, const Derived& _m, const IOFormat& f
if (fmt.precision == StreamPrecision) {
explicit_precision = 0;
} else if (fmt.precision == FullPrecision) {
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsInteger) { explicit_precision = 0; }
else {
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsInteger) {
explicit_precision = 0;
} else {
explicit_precision = significant_decimals_impl<Scalar>::run();
}
} else {
+4 -4
View File
@@ -199,13 +199,13 @@ class IndexedViewImpl<XprType, RowIndices, ColIndices, StorageKind, true>
IndexedViewImpl(XprType& xpr, const T0& rowIndices, const T1& colIndices) : Base(xpr, rowIndices, colIndices) {}
Index rowIncrement() const {
EIGEN_IF_CONSTEXPR(traits<Derived>::RowIncr != DynamicIndex && traits<Derived>::RowIncr != Undefined) {
EIGEN_IF_CONSTEXPR (traits<Derived>::RowIncr != DynamicIndex && traits<Derived>::RowIncr != Undefined) {
return traits<Derived>::RowIncr;
}
return IndexedViewHelper<RowIndices>::incr(this->rowIndices());
}
Index colIncrement() const {
EIGEN_IF_CONSTEXPR(traits<Derived>::ColIncr != DynamicIndex && traits<Derived>::ColIncr != Undefined) {
EIGEN_IF_CONSTEXPR (traits<Derived>::ColIncr != DynamicIndex && traits<Derived>::ColIncr != Undefined) {
return traits<Derived>::ColIncr;
}
return IndexedViewHelper<ColIndices>::incr(this->colIndices());
@@ -228,14 +228,14 @@ class IndexedViewImpl<XprType, RowIndices, ColIndices, StorageKind, true>
}
EIGEN_DEVICE_FUNC constexpr Index innerStride() const noexcept {
EIGEN_IF_CONSTEXPR(traits<Derived>::InnerStrideAtCompileTime != Dynamic) {
EIGEN_IF_CONSTEXPR (traits<Derived>::InnerStrideAtCompileTime != Dynamic) {
return traits<Derived>::InnerStrideAtCompileTime;
}
return innerIncrement() * this->nestedExpression().innerStride();
}
EIGEN_DEVICE_FUNC constexpr Index outerStride() const noexcept {
EIGEN_IF_CONSTEXPR(traits<Derived>::OuterStrideAtCompileTime != Dynamic) {
EIGEN_IF_CONSTEXPR (traits<Derived>::OuterStrideAtCompileTime != Dynamic) {
return traits<Derived>::OuterStrideAtCompileTime;
}
return outerIncrement() * this->nestedExpression().outerStride();
+1 -1
View File
@@ -921,7 +921,7 @@ struct copysign_impl<Scalar, true, IsInteger> {
template <typename Scalar>
struct copysign_impl<Scalar, false, true> {
EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& a, const Scalar& b) {
EIGEN_IF_CONSTEXPR(!NumTraits<Scalar>::IsSigned) return a;
EIGEN_IF_CONSTEXPR (!NumTraits<Scalar>::IsSigned) return a;
const Scalar abs_a = a < Scalar(0) ? -a : a;
return b < Scalar(0) ? -abs_a : abs_a;
}
+1 -1
View File
@@ -203,7 +203,7 @@ struct evaluator<PartialReduxExpr<ArgType, MemberOp, Direction> >
// Workaround for issue 1612 (closed): when PacketSize==1 (i.e. complex<double> with 128bits registers) the
// storage-order of panel gets reversed and methods like packetByOuterInner do not make sense in this context, so
// bypass "vectorization":
EIGEN_IF_CONSTEXPR(PacketSize == 1) return internal::pset1<PacketType>(coeff(idx));
EIGEN_IF_CONSTEXPR (PacketSize == 1) return internal::pset1<PacketType>(coeff(idx));
Index startRow = Direction == Vertical ? 0 : idx;
Index startCol = Direction == Vertical ? idx : 0;
+15 -13
View File
@@ -171,8 +171,9 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
*
* See DenseCoeffsBase<Derived,ReadOnlyAccessors>::coeff(Index) const for details. */
EIGEN_DEVICE_FUNC constexpr const Scalar& coeff(Index rowId, Index colId) const {
EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; }
else { // column-major
EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) {
return m_storage.data()[colId + rowId * m_storage.cols()];
} else { // column-major
return m_storage.data()[rowId + colId * m_storage.rows()];
}
}
@@ -188,8 +189,9 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
*
* See DenseCoeffsBase<Derived,WriteAccessors>::coeffRef(Index,Index) const for details. */
EIGEN_DEVICE_FUNC constexpr Scalar& coeffRef(Index rowId, Index colId) {
EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; }
else { // column-major
EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) {
return m_storage.data()[colId + rowId * m_storage.cols()];
} else { // column-major
return m_storage.data()[rowId + colId * m_storage.rows()];
}
}
@@ -203,8 +205,9 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
/** This is the const version of coeffRef(Index,Index) which is thus synonym of coeff(Index,Index).
* It is provided for convenience. */
EIGEN_DEVICE_FUNC constexpr const Scalar& coeffRef(Index rowId, Index colId) const {
EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; }
else { // column-major
EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) {
return m_storage.data()[colId + rowId * m_storage.cols()];
} else { // column-major
return m_storage.data()[rowId + colId * m_storage.rows()];
}
}
@@ -303,8 +306,9 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
#ifdef EIGEN_INITIALIZE_COEFFS
bool size_changed = size != this->size();
#endif
EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { m_storage.resize(size, 1, size); }
else {
EIGEN_IF_CONSTEXPR (RowsAtCompileTime == 1) {
m_storage.resize(size, 1, size);
} else {
m_storage.resize(size, size, 1);
}
#ifdef EIGEN_INITIALIZE_COEFFS
@@ -347,15 +351,13 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
other.rows(), other.cols());
#endif
const Index othersize = other.rows() * other.cols();
EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) {
EIGEN_IF_CONSTEXPR (RowsAtCompileTime == 1) {
eigen_assert(other.rows() == 1 || other.cols() == 1);
resize(1, othersize);
}
else EIGEN_IF_CONSTEXPR(ColsAtCompileTime == 1) {
} else EIGEN_IF_CONSTEXPR (ColsAtCompileTime == 1) {
eigen_assert(other.rows() == 1 || other.cols() == 1);
resize(othersize, 1);
}
else {
} else {
resize(other.rows(), other.cols());
}
}
+30 -36
View File
@@ -899,16 +899,15 @@ struct triangular_diagonal_product_impl {
const Index rows = matrix.rows();
const Index cols = matrix.cols();
for (Index col = 0; col < cols; ++col) {
EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) {
EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) {
const Index end = (std::min)(rows, ((Mode & (UnitDiag | ZeroDiag)) ? col : col + 1));
addStoredSegment(dst, matrix, diagonal, 0, end, col, alpha);
}
else {
} else {
const Index begin = ((Mode & (UnitDiag | ZeroDiag)) ? col + 1 : col);
addStoredSegment(dst, matrix, diagonal, begin, rows - begin, col, alpha);
}
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) {
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) {
if (col < rows) addUnitCoeff(dst, diagonal, col, alpha);
}
}
@@ -928,10 +927,9 @@ struct triangular_diagonal_product_impl {
template <typename Dest, typename Alpha>
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addUnitCoeff(Dest& dst, const DiagonalType& diagonal, Index index,
const Alpha& alpha) {
EIGEN_IF_CONSTEXPR(ProductOrder == OnTheLeft) {
EIGEN_IF_CONSTEXPR (ProductOrder == OnTheLeft) {
dst.coeffRef(index, index) += alpha * (diagonal.coeff(index) * MatrixScalar(1));
}
else {
} else {
dst.coeffRef(index, index) += alpha * (MatrixScalar(1) * diagonal.coeff(index));
}
}
@@ -1041,10 +1039,9 @@ struct selfadjoint_diagonal_product_impl {
// Stored half: one column-strided segment per output column.
for (Index col = 0; col < size; ++col) {
EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) {
EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) {
storedSegment<Accumulate>(dst, matrix, diagonal, 0, col + 1, col, alpha);
}
else {
} else {
storedSegment<Accumulate>(dst, matrix, diagonal, col, size - col, col, alpha);
}
}
@@ -1053,7 +1050,7 @@ struct selfadjoint_diagonal_product_impl {
for (Index ib = 0; ib < size; ib += BlockSize) {
const Index ib_end = numext::mini(size, ib + BlockSize);
const Index br = ib_end - ib;
EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) {
EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) {
// Off-diagonal: write strict-lower of dst from strict-upper of source.
for (Index jb = 0; jb < ib; jb += BlockSize) {
const Index bc = numext::mini(jb + BlockSize, ib) - jb;
@@ -1062,8 +1059,7 @@ struct selfadjoint_diagonal_product_impl {
// Diagonal tile: in-tile strict-lower mirror.
for (Index col = ib; col < ib_end; ++col)
conjugateSegment<Accumulate>(dst, matrix, diagonal, col + 1, ib_end - col - 1, col, alpha);
}
else {
} else {
// Off-diagonal: write strict-upper of dst from strict-lower of source.
for (Index jb = ib_end; jb < size; jb += BlockSize) {
const Index bc = numext::mini(size, jb + BlockSize) - jb;
@@ -1083,10 +1079,9 @@ struct selfadjoint_diagonal_product_impl {
if (size <= 0) return;
auto dstSegment = dst.col(col).segment(begin, size);
auto srcSegment = matrix.col(col).segment(begin, size);
EIGEN_IF_CONSTEXPR(Accumulate) {
EIGEN_IF_CONSTEXPR (Accumulate) {
diagonal_product_segment_impl<ProductOrder>::run(dstSegment, srcSegment, diagonal, begin, col, alpha);
}
else {
} else {
diagonal_product_segment_impl<ProductOrder>::runOverwrite(dstSegment, srcSegment, diagonal, begin, col);
}
}
@@ -1098,10 +1093,9 @@ struct selfadjoint_diagonal_product_impl {
if (size <= 0) return;
auto dstSegment = dst.col(col).segment(begin, size);
auto srcSegment = matrix.row(col).segment(begin, size).conjugate().transpose();
EIGEN_IF_CONSTEXPR(Accumulate) {
EIGEN_IF_CONSTEXPR (Accumulate) {
diagonal_product_segment_impl<ProductOrder>::run(dstSegment, srcSegment, diagonal, begin, col, alpha);
}
else {
} else {
diagonal_product_segment_impl<ProductOrder>::runOverwrite(dstSegment, srcSegment, diagonal, begin, col);
}
}
@@ -1115,17 +1109,18 @@ struct selfadjoint_diagonal_product_impl {
Index br, Index bc, const Alpha& alpha) {
auto dstBlock = dst.block(ib, jb, br, bc);
auto srcAdjoint = matrix.block(jb, ib, bc, br).adjoint();
EIGEN_IF_CONSTEXPR(ProductOrder == OnTheRight) {
EIGEN_IF_CONSTEXPR (ProductOrder == OnTheRight) {
auto scaled = srcAdjoint * diagonal.segment(jb, bc).asDiagonal();
EIGEN_IF_CONSTEXPR(Accumulate) { dstBlock.noalias() += alpha * scaled; }
else {
EIGEN_IF_CONSTEXPR (Accumulate) {
dstBlock.noalias() += alpha * scaled;
} else {
dstBlock.noalias() = scaled;
}
}
else {
} else {
auto scaled = diagonal.segment(ib, br).asDiagonal() * srcAdjoint;
EIGEN_IF_CONSTEXPR(Accumulate) { dstBlock.noalias() += alpha * scaled; }
else {
EIGEN_IF_CONSTEXPR (Accumulate) {
dstBlock.noalias() += alpha * scaled;
} else {
dstBlock.noalias() = scaled;
}
}
@@ -1180,13 +1175,12 @@ struct generic_product_impl<Lhs, Rhs, SelfAdjointShape, DiagonalShape, ProductTa
template <typename Dest>
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
EIGEN_IF_CONSTEXPR(LhsBlasTraits::HasScalarFactor) {
EIGEN_IF_CONSTEXPR (LhsBlasTraits::HasScalarFactor) {
// Folded scalar factor present: zero dst then accumulate at the extracted alpha.
Scalar factor = LhsBlasTraits::extractScalarFactor(lhs.nestedExpression());
dst.setZero();
Kernel::run(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal(), factor);
}
else {
} else {
// No scalar factor: kernel writes every entry exactly once, skip setZero.
Kernel::runOverwrite(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal());
}
@@ -1219,12 +1213,11 @@ struct generic_product_impl<Lhs, Rhs, DiagonalShape, SelfAdjointShape, ProductTa
template <typename Dest>
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
EIGEN_IF_CONSTEXPR(RhsBlasTraits::HasScalarFactor) {
EIGEN_IF_CONSTEXPR (RhsBlasTraits::HasScalarFactor) {
Scalar factor = RhsBlasTraits::extractScalarFactor(rhs.nestedExpression());
dst.setZero();
Kernel::run(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal(), factor);
}
else {
} else {
Kernel::runOverwrite(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal());
}
}
@@ -1471,11 +1464,11 @@ struct triangular_diagonal_product_lazy_evaluator_base : evaluator_base<Derived>
const bool inActive = ((Mode & Upper) == Upper) ? (row <= col) : (row >= col);
if (!inActive) return Scalar(0);
if (row == col) {
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) {
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) {
return ProductOrder == OnTheLeft ? Scalar(m_diagImpl.coeff(row) * MatrixScalar(1))
: Scalar(MatrixScalar(1) * m_diagImpl.coeff(col));
}
EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag) return Scalar(0);
EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag) return Scalar(0);
}
return ProductOrder == OnTheLeft ? Scalar(m_diagImpl.coeff(row) * m_matImpl.coeff(row, col))
: Scalar(m_matImpl.coeff(row, col) * m_diagImpl.coeff(col));
@@ -1701,8 +1694,9 @@ struct transposition_matrix_product {
for (Index k = (Transposed ? size - 1 : 0); Transposed ? k >= 0 : k < size; Transposed ? --k : ++k)
if (Index(j = tr.coeff(k)) != k) {
EIGEN_IF_CONSTEXPR(Side == OnTheLeft) { dst.row(k).swap(dst.row(j)); }
else EIGEN_IF_CONSTEXPR(Side == OnTheRight) {
EIGEN_IF_CONSTEXPR (Side == OnTheLeft) {
dst.row(k).swap(dst.row(j));
} else EIGEN_IF_CONSTEXPR (Side == OnTheRight) {
dst.col(k).swap(dst.col(j));
}
}
+2 -3
View File
@@ -126,12 +126,11 @@ class RefBase : public MapBase<Derived> {
// Determine runtime rows and columns.
Index rows = expr.rows();
Index cols = expr.cols();
EIGEN_IF_CONSTEXPR(PlainObjectType::RowsAtCompileTime == 1) {
EIGEN_IF_CONSTEXPR (PlainObjectType::RowsAtCompileTime == 1) {
eigen_assert(expr.rows() == 1 || expr.cols() == 1);
rows = 1;
cols = expr.size();
}
else EIGEN_IF_CONSTEXPR(PlainObjectType::ColsAtCompileTime == 1) {
} else EIGEN_IF_CONSTEXPR (PlainObjectType::ColsAtCompileTime == 1) {
eigen_assert(expr.rows() == 1 || expr.cols() == 1);
rows = expr.size();
cols = 1;
+2 -3
View File
@@ -296,11 +296,10 @@ struct reshaped_evaluator<ArgType, Rows, Cols, Order, /* HasDirectAccess */ fals
typedef std::pair<Index, Index> RowCol;
EIGEN_DEVICE_FUNC constexpr inline RowCol index_remap(Index rowId, Index colId) const {
EIGEN_IF_CONSTEXPR(Order == ColMajor) {
EIGEN_IF_CONSTEXPR (Order == ColMajor) {
const Index nth_elem_idx = colId * m_xpr.rows() + rowId;
return RowCol(nth_elem_idx % m_xpr.nestedExpression().rows(), nth_elem_idx / m_xpr.nestedExpression().rows());
}
else {
} else {
const Index nth_elem_idx = colId + rowId * m_xpr.cols();
return RowCol(nth_elem_idx / m_xpr.nestedExpression().cols(), nth_elem_idx % m_xpr.nestedExpression().cols());
}
+2 -3
View File
@@ -219,11 +219,10 @@ class SelfAdjointView : public TriangularBase<SelfAdjointView<MatrixType_, UpLo>
const Index n = m_matrix.rows();
for (Index col = 0; col < n; ++col) {
RealScalar_ abs_col_sum;
EIGEN_IF_CONSTEXPR(UpLo == Lower) {
EIGEN_IF_CONSTEXPR (UpLo == Lower) {
abs_col_sum =
m_matrix.col(col).tail(n - col).template lpNorm<1>() + m_matrix.row(col).head(col).template lpNorm<1>();
}
else {
} else {
abs_col_sum =
m_matrix.col(col).head(col).template lpNorm<1>() + m_matrix.row(col).tail(n - col).template lpNorm<1>();
}
+19 -19
View File
@@ -391,11 +391,11 @@ class TriangularView
/** \returns the determinant of the triangular matrix
* \sa MatrixBase::determinant() */
EIGEN_DEVICE_FUNC Scalar determinant() const {
EIGEN_IF_CONSTEXPR(Mode & UnitDiag) { return 1; }
else EIGEN_IF_CONSTEXPR(Mode & ZeroDiag) {
EIGEN_IF_CONSTEXPR (Mode & UnitDiag) {
return 1;
} else EIGEN_IF_CONSTEXPR (Mode & ZeroDiag) {
return 0;
}
else {
} else {
return m_matrix.diagonal().prod();
}
}
@@ -777,18 +777,20 @@ class triangular_dense_assignment_kernel
#endif
EIGEN_DEVICE_FUNC void assignDiagonalCoeff(Index id) {
EIGEN_IF_CONSTEXPR(Mode == UnitDiag && SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(1)); }
else EIGEN_IF_CONSTEXPR(Mode == ZeroDiag && SetOpposite) {
EIGEN_IF_CONSTEXPR (Mode == UnitDiag && SetOpposite) {
m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(1));
} else EIGEN_IF_CONSTEXPR (Mode == ZeroDiag && SetOpposite) {
m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(0));
}
else EIGEN_IF_CONSTEXPR(Mode == 0) {
} else EIGEN_IF_CONSTEXPR (Mode == 0) {
Base::assignCoeff(id, id);
}
}
EIGEN_DEVICE_FUNC void assignOppositeCoeff(Index row, Index col) {
eigen_internal_assert(row != col);
EIGEN_IF_CONSTEXPR(SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(row, col), Scalar(0)); }
EIGEN_IF_CONSTEXPR (SetOpposite) {
m_functor.assignCoeff(m_dst.coeffRef(row, col), Scalar(0));
}
}
};
@@ -894,8 +896,9 @@ struct triangular_assignment_loop {
kernel.assignDiagonalCoeff(row);
else if (((Mode & Lower) && row > col) || ((Mode & Upper) && row < col))
kernel.assignCoeff(row, col);
else
EIGEN_IF_CONSTEXPR(SetOpposite) { kernel.assignOppositeCoeff(row, col); }
else EIGEN_IF_CONSTEXPR (SetOpposite) {
kernel.assignOppositeCoeff(row, col);
}
}
};
@@ -942,22 +945,19 @@ struct triangular_assignment_loop<Kernel, Mode, Dynamic, SetOpposite> {
const Index maxi = numext::mini(outer, innerSize);
Index i = 0;
EIGEN_IF_CONSTEXPR(ActiveBeforeDiag) {
EIGEN_IF_CONSTEXPR (ActiveBeforeDiag) {
for (; i < maxi; ++i) kernel.assignCoeff(row(outer, i), col(outer, i));
}
else EIGEN_IF_CONSTEXPR(SetOpposite) {
} else EIGEN_IF_CONSTEXPR (SetOpposite) {
for (; i < maxi; ++i) kernel.assignOppositeCoeff(row(outer, i), col(outer, i));
}
else {
} else {
i = maxi;
}
if (i < innerSize) kernel.assignDiagonalCoeff(i++);
EIGEN_IF_CONSTEXPR(!ActiveBeforeDiag) {
EIGEN_IF_CONSTEXPR (!ActiveBeforeDiag) {
for (; i < innerSize; ++i) kernel.assignCoeff(row(outer, i), col(outer, i));
}
else EIGEN_IF_CONSTEXPR(SetOpposite) {
} else EIGEN_IF_CONSTEXPR (SetOpposite) {
for (; i < innerSize; ++i) kernel.assignOppositeCoeff(row(outer, i), col(outer, i));
}
}
+31 -40
View File
@@ -1001,10 +1001,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
Index count = 0;
const Index peeled_k = (depth / PacketSize) * PacketSize;
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
// skip what we have before
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
@@ -1014,7 +1014,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
const LinearMapper dm6 = rhs.getLinearMapper(0, j2 + 6);
const LinearMapper dm7 = rhs.getLinearMapper(0, j2 + 7);
Index k = 0;
EIGEN_IF_CONSTEXPR((PacketSize % 8) == 0 || PacketSize == 4) {
EIGEN_IF_CONSTEXPR ((PacketSize % 8) == 0 || PacketSize == 4) {
for (; k < peeled_k; k += PacketSize) {
PacketBlock<Packet, 8> kernel;
@@ -1027,7 +1027,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
kernel.packet[6] = dm6.template loadPacket<Packet>(k);
kernel.packet[7] = dm7.template loadPacket<Packet>(k);
EIGEN_IF_CONSTEXPR(PacketSize == 4) {
EIGEN_IF_CONSTEXPR (PacketSize == 4) {
// For PacketSize==4 we cannot ptranspose 8 packets directly; compose two
// 4-packet transposes (cols 0-3 and 4-7) and interleave the halves so
// the 8 stores produce 4 rows of 8 packed elements.
@@ -1051,8 +1051,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
kernel.packet[5] = tmp_hi.packet[2];
kernel.packet[6] = tmp_lo.packet[3];
kernel.packet[7] = tmp_hi.packet[3];
}
else {
} else {
ptranspose(kernel);
}
@@ -1079,28 +1078,28 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
count += 8;
}
// skip what we have after
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
}
}
EIGEN_IF_CONSTEXPR(nr >= 4) {
EIGEN_IF_CONSTEXPR (nr >= 4) {
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
// skip what we have before
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3);
Index k = 0;
EIGEN_IF_CONSTEXPR((PacketSize % 4) == 0 || PacketSize == 2) {
EIGEN_IF_CONSTEXPR ((PacketSize % 4) == 0 || PacketSize == 2) {
for (; k < peeled_k; k += PacketSize) {
PacketBlock<Packet, 4> kernel;
kernel.packet[0] = dm0.template loadPacket<Packet>(k);
kernel.packet[1] = dm1.template loadPacket<Packet>(k);
kernel.packet[2] = dm2.template loadPacket<Packet>(k);
kernel.packet[3] = dm3.template loadPacket<Packet>(k);
EIGEN_IF_CONSTEXPR(PacketSize == 2) {
EIGEN_IF_CONSTEXPR (PacketSize == 2) {
// See the matching note in GeneralBlockPanelKernel.h.
PacketBlock<Packet, 2> tmp01;
tmp01.packet[0] = kernel.packet[0];
@@ -1114,8 +1113,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
kernel.packet[1] = tmp23.packet[0];
kernel.packet[2] = tmp01.packet[1];
kernel.packet[3] = tmp23.packet[1];
}
else {
} else {
ptranspose(kernel);
}
pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0]));
@@ -1133,19 +1131,19 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
count += 4;
}
// skip what we have after
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
}
}
// copy the remaining columns one at a time (nr==1)
for (Index j2 = packet_cols4; j2 < cols; ++j2) {
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
const LinearMapper dm0 = rhs.getLinearMapper(0, j2);
for (Index k = 0; k < depth; k++) {
blockB[count] = cj(dm0(k));
count += 1;
}
EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
}
}
@@ -1174,33 +1172,29 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, 8, RowMajor, Conjugate, PanelMod
Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
Index count = 0;
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
// skip what we have before
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
for (Index k = 0; k < depth; k++) {
EIGEN_IF_CONSTEXPR(PacketSize == 8) {
EIGEN_IF_CONSTEXPR (PacketSize == 8) {
// Packet A = ploadu<Packet>(&rhs.data()[k*rhs.stride() + j2]);
Packet A = rhs.template loadPacket<Packet>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
}
else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 8) {
} else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 8) {
HalfPacket A = rhs.template loadPacket<HalfPacket>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
}
else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 8) {
} else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 8) {
QuarterPacket A = rhs.template loadPacket<QuarterPacket>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
}
else EIGEN_IF_CONSTEXPR(PacketSize == 4) {
} else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
// Packet A = ploadu<Packet>(&rhs.data()[k*rhs.stride() + j2]);
// Packet B = ploadu<Packet>(&rhs.data()[k*rhs.stride() + j2 + PacketSize]);
Packet A = rhs.template loadPacket<Packet>(k, j2);
Packet B = rhs.template loadPacket<Packet>(k, j2 + PacketSize);
pstoreu(blockB + count, cj.pconj(A));
pstoreu(blockB + count + PacketSize, cj.pconj(B));
}
else {
} else {
// const Scalar* b0 = &rhs.data()[k*rhs.stride() + j2];
const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
blockB[count + 0] = cj(dm0(0));
@@ -1215,31 +1209,28 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, 8, RowMajor, Conjugate, PanelMod
count += 8;
}
// skip what we have after
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
}
}
EIGEN_IF_CONSTEXPR(nr >= 4) {
EIGEN_IF_CONSTEXPR (nr >= 4) {
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
// skip what we have before
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
for (Index k = 0; k < depth; k++) {
EIGEN_IF_CONSTEXPR(PacketSize == 4) {
EIGEN_IF_CONSTEXPR (PacketSize == 4) {
Packet A = rhs.template loadPacket<Packet>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
count += PacketSize;
}
else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 4) {
} else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 4) {
HalfPacket A = rhs.template loadPacket<HalfPacket>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
count += HalfPacketSize;
}
else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 4) {
} else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 4) {
QuarterPacket A = rhs.template loadPacket<QuarterPacket>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
count += QuarterPacketSize;
}
else {
} else {
const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
blockB[count + 0] = cj(dm0(0));
blockB[count + 1] = cj(dm0(1));
@@ -1249,17 +1240,17 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, 8, RowMajor, Conjugate, PanelMod
}
}
// skip what we have after
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
}
}
// copy the remaining columns one at a time (nr==1)
for (Index j2 = packet_cols4; j2 < cols; ++j2) {
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
for (Index k = 0; k < depth; k++) {
blockB[count] = cj(rhs(k, j2));
count += 1;
}
EIGEN_IF_CONSTEXPR(PanelMode) count += stride - offset - depth;
EIGEN_IF_CONSTEXPR (PanelMode) count += stride - offset - depth;
}
}
};
+198 -156
View File
@@ -136,23 +136,22 @@ EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_
static_assert(unrollM == EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW");
urolls::template transpose<unrollN, 0>(zmm);
EIGEN_IF_CONSTEXPR(unrollN > U2) urolls::template transpose<unrollN, 2>(zmm);
EIGEN_IF_CONSTEXPR(unrollN > U1) urolls::template transpose<unrollN, 1>(zmm);
EIGEN_IF_CONSTEXPR (unrollN > U2) urolls::template transpose<unrollN, 2>(zmm);
EIGEN_IF_CONSTEXPR (unrollN > U1) urolls::template transpose<unrollN, 1>(zmm);
static_assert((remN && unrollN == U1) || !remN, "When handling N remainder set unrollN=U1");
EIGEN_IF_CONSTEXPR(!remN) {
EIGEN_IF_CONSTEXPR (!remN) {
urolls::template storeC<std::min(unrollN, U1), unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
EIGEN_IF_CONSTEXPR(unrollN > U1) {
EIGEN_IF_CONSTEXPR (unrollN > U1) {
constexpr int64_t unrollN_ = std::min(unrollN - U1, U1);
urolls::template storeC<unrollN_, unrollN, 1, remM>(C_arr + U1 * LDC, LDC, zmm, remM_);
}
EIGEN_IF_CONSTEXPR(unrollN > U2) {
EIGEN_IF_CONSTEXPR (unrollN > U2) {
constexpr int64_t unrollN_ = std::min(unrollN - U2, U1);
urolls::template storeC<unrollN_, unrollN, 2, remM>(C_arr + U2 * LDC, LDC, zmm, remM_);
}
}
else {
EIGEN_IF_CONSTEXPR((std::is_same<Scalar, float>::value)) {
} else {
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, float>::value)) {
// Note: without "if constexpr" this section of code will also be
// parsed by the compiler so each of the storeC will still be instantiated.
// We use enable_if in aux_storeC to set it to an empty function for
@@ -187,8 +186,7 @@ EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_
urolls::template storeC<2, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
else if (remN_ == 1)
urolls::template storeC<1, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
}
else {
} else {
if (remN_ == 7)
urolls::template storeC<7, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
else if (remN_ == 6)
@@ -244,23 +242,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 3, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 3, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_B_LOAD_SETS * 3,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, false, false>(zmm, &C_arr[i + j * LDC], LDC);
}
}
@@ -273,23 +274,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 3, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_B_LOAD_SETS * 3,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 3, 4, 1, EIGEN_AVX_B_LOAD_SETS * 3, EIGEN_AVX_MAX_A_BCAST>(
B_t, A_t, LDB, LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<3, 4>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<3, 4>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, true, false>(zmm, &C_arr[i + j * LDC], LDC, 4);
}
i += 4;
@@ -303,23 +307,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 3, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_B_LOAD_SETS * 3,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 3, 2, 1, EIGEN_AVX_B_LOAD_SETS * 3, EIGEN_AVX_MAX_A_BCAST>(
B_t, A_t, LDB, LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<3, 2>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<3, 2>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, true, false>(zmm, &C_arr[i + j * LDC], LDC, 2);
}
i += 2;
@@ -334,22 +341,25 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 3, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_B_LOAD_SETS * 3, 1>(
B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 3, 1, 1, EIGEN_AVX_B_LOAD_SETS * 3, 1>(B_t, A_t, LDB, LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<3, 1>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<3, 1>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, true, false>(zmm, &C_arr[i + j * LDC], LDC, 1);
}
}
@@ -360,30 +370,33 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
int64_t i = 0;
for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) {
Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j];
EIGEN_IF_CONSTEXPR(isCRowMajor) B_t = &B_arr[0 * LDB + j];
EIGEN_IF_CONSTEXPR (isCRowMajor) B_t = &B_arr[0 * LDB + j];
PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
urolls::template setzero<2, EIGEN_AVX_MAX_NUM_ROW>(zmm);
for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
urolls::template microKernel<isARowMajor, 2, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 2, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, false, false>(zmm, &C_arr[i + j * LDC], LDC);
}
}
@@ -396,23 +409,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 2, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 2, 4, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<2, 4>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<2, 4>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, true, false>(zmm, &C_arr[i + j * LDC], LDC, 4);
}
i += 4;
@@ -426,23 +442,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 2, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 2, 2, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<2, 2>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<2, 2>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, true, false>(zmm, &C_arr[i + j * LDC], LDC, 2);
}
i += 2;
@@ -456,22 +475,25 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 2, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD, 1>(B_t, A_t, LDB,
LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 2, 1, 1, EIGEN_AVX_MAX_B_LOAD, 1>(B_t, A_t, LDB, LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<2, 1>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<2, 1>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, true, false>(zmm, &C_arr[i + j * LDC], LDC, 1);
}
}
@@ -488,23 +510,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_B_LOAD_SETS * 1,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, false, false>(zmm, &C_arr[i + j * LDC], LDC);
}
}
@@ -517,23 +542,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 1, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 1, 4, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<1, 4>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<1, 4>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, false>(zmm, &C_arr[i + j * LDC], LDC, 4);
}
i += 4;
@@ -547,23 +575,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 1, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 1, 2, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<1, 2>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<1, 2>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, false>(zmm, &C_arr[i + j * LDC], LDC, 2);
}
i += 2;
@@ -578,22 +609,25 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 1, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD, 1>(B_t, A_t, LDB,
LDA, zmm);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 1, 1, 1, EIGEN_AVX_B_LOAD_SETS * 1, 1>(B_t, A_t, LDB, LDA, zmm);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<1, 1>(&C_arr[i * LDC + j], LDC, zmm);
urolls::template storeC<1, 1>(&C_arr[i * LDC + j], LDC, zmm);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, false>(zmm, &C_arr[i + j * LDC], LDC, 1);
}
}
@@ -612,23 +646,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, false, true>(zmm, &C_arr[i + j * LDC], LDC, 0, N - j);
}
}
@@ -641,23 +678,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 1, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 1, 4, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST, true>(
B_t, A_t, LDB, LDA, zmm, N - j);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
urolls::template storeC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, true>(zmm, &C_arr[i + j * LDC], LDC, 4, N - j);
}
i += 4;
@@ -671,23 +711,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 1, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 1, 2, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST, true>(
B_t, A_t, LDB, LDA, zmm, N - j);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
urolls::template storeC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, true>(zmm, &C_arr[i + j * LDC], LDC, 2, N - j);
}
i += 2;
@@ -701,23 +744,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
urolls::template microKernel<isARowMajor, 1, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD, 1, true>(
B_t, A_t, LDB, LDA, zmm, N - j);
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t += EIGEN_AVX_MAX_K_UNROL;
else
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
}
EIGEN_IF_CONSTEXPR(handleKRem) {
EIGEN_IF_CONSTEXPR (handleKRem) {
for (int64_t k = K_; k < K; k++) {
urolls::template microKernel<isARowMajor, 1, 1, 1, EIGEN_AVX_MAX_B_LOAD, 1, true>(B_t, A_t, LDB, LDA, zmm,
N - j);
B_t += LDB;
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
else A_t += LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
A_t++;
else
A_t += LDA;
}
}
EIGEN_IF_CONSTEXPR(isCRowMajor) {
EIGEN_IF_CONSTEXPR (isCRowMajor) {
urolls::template updateC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
urolls::template storeC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
}
else {
} else {
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, true>(zmm, &C_arr[i + j * LDC], LDC, 1, N - j);
}
}
@@ -840,7 +886,7 @@ EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar* B_arr, int64_t LDB, int64_t K,
B_temp += U1;
k += U1;
}
EIGEN_IF_CONSTEXPR(U1 > 8) {
EIGEN_IF_CONSTEXPR (U1 > 8) {
// Note: without "if constexpr" this section of code will also be
// parsed by the compiler so there is an additional check in {load/store}BBlock
// to make sure the counter is not non-negative.
@@ -850,7 +896,7 @@ EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar* B_arr, int64_t LDB, int64_t K,
k += 8;
}
}
EIGEN_IF_CONSTEXPR(U1 > 4) {
EIGEN_IF_CONSTEXPR (U1 > 4) {
// Note: without "if constexpr" this section of code will also be
// parsed by the compiler so there is an additional check in {load/store}BBlock
// to make sure the counter is not non-negative.
@@ -921,7 +967,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
int64_t sizeBTemp = 0;
Scalar* B_temp = NULL;
EIGEN_IF_CONSTEXPR(!isBRowMajor) {
EIGEN_IF_CONSTEXPR (!isBRowMajor) {
/**
* If B is col-major, we copy it to a fixed-size temporary array of size at most ~numM*kB and
* transpose it to row-major. Call the solve routine, and copy+transpose it back to the original array.
@@ -930,7 +976,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
sizeBTemp = (((std::min(kB, numRHS) + psize - 1) / psize + 4) * psize) * numM;
}
EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar*)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64);
EIGEN_IF_CONSTEXPR (!isBRowMajor) B_temp = (Scalar*)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64);
for (int64_t k = 0; k < numRHS; k += kB) {
int64_t bK = numRHS - k > kB ? kB : numRHS - k;
@@ -944,7 +990,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
int64_t LDT = ((bkL + (numScalarPerCache - 1)) / numScalarPerCache) * numScalarPerCache;
int64_t offsetBTemp = 0;
for (int64_t i = 0; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) {
EIGEN_IF_CONSTEXPR(!isBRowMajor) {
EIGEN_IF_CONSTEXPR (!isBRowMajor) {
int64_t indA_i = isFWDSolve ? i : M - 1 - i;
int64_t indB_i = isFWDSolve ? i : M - (i + EIGEN_AVX_MAX_NUM_ROW);
int64_t offB_1 = isFWDSolve ? offsetBTemp : sizeBTemp - EIGEN_AVX_MAX_NUM_ROW * LDT - offsetBTemp;
@@ -958,8 +1004,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
copyBToRowMajor<Scalar, false, false>(B_arr + indB_i + k * LDB, LDB, bK, B_temp + offB_1, LDT);
offsetBTemp += EIGEN_AVX_MAX_NUM_ROW * LDT;
}
else {
} else {
int64_t ind = isFWDSolve ? i : M - 1 - i;
triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(
&A_arr[idA<isARowMajor>(ind, ind, LDA)], B_arr + k + ind * LDB, EIGEN_AVX_MAX_NUM_ROW, bK, LDA, LDB);
@@ -977,7 +1022,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
* |__|__|__|__ |__|
* |********|__| |**|
*/
EIGEN_IF_CONSTEXPR(isBRowMajor) {
EIGEN_IF_CONSTEXPR (isBRowMajor) {
int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW);
int64_t indA_j = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW);
int64_t indB_i = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW);
@@ -985,8 +1030,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
gemmKernel<Scalar, isARowMajor, isBRowMajor, false, false>(
&A_arr[idA<isARowMajor>(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB,
EIGEN_AVX_MAX_NUM_ROW, bK, i + EIGEN_AVX_MAX_NUM_ROW, LDA, LDB, LDB);
}
else {
} else {
if (offsetBTemp + EIGEN_AVX_MAX_NUM_ROW * LDT > sizeBTemp) {
/**
* Similar idea as mentioned above, but here we are limited by the number of updated values of B
@@ -1024,7 +1068,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
int64_t bM = M - M_;
if (bM > 0) {
if (M_ > 0) {
EIGEN_IF_CONSTEXPR(isBRowMajor) {
EIGEN_IF_CONSTEXPR (isBRowMajor) {
int64_t indA_i = isFWDSolve ? M_ : 0;
int64_t indA_j = isFWDSolve ? 0 : bM;
int64_t indB_i = isFWDSolve ? 0 : bM;
@@ -1032,8 +1076,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
gemmKernel<Scalar, isARowMajor, isBRowMajor, false, false>(
&A_arr[idA<isARowMajor>(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB, bM,
bK, M_, LDA, LDB, LDB);
}
else {
} else {
int64_t indA_i = isFWDSolve ? M_ : 0;
int64_t indA_j = isFWDSolve ? gemmOff : bM;
int64_t indB_i = isFWDSolve ? M_ : 0;
@@ -1043,7 +1086,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
M_ - gemmOff, LDA, LDT, LDB);
}
}
EIGEN_IF_CONSTEXPR(!isBRowMajor) {
EIGEN_IF_CONSTEXPR (!isBRowMajor) {
int64_t indA_i = isFWDSolve ? M_ : M - 1 - M_;
int64_t indB_i = isFWDSolve ? M_ : 0;
int64_t offB_1 = isFWDSolve ? 0 : (bM - 1) * bkL;
@@ -1051,8 +1094,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(&A_arr[idA<isARowMajor>(indA_i, indA_i, LDA)],
B_temp + offB_1, bM, bkL, LDA, bkL);
copyBToRowMajor<Scalar, false, true>(B_arr + indB_i + k * LDB, LDB, bK, B_temp, bkL, bM);
}
else {
} else {
int64_t ind = isFWDSolve ? M_ : M - 1 - M_;
triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(&A_arr[idA<isARowMajor>(ind, ind, LDA)],
B_arr + k + ind * LDB, bM, bK, LDA, LDB);
@@ -1060,7 +1102,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
}
}
EIGEN_IF_CONSTEXPR(!isBRowMajor) handmade_aligned_free(B_temp);
EIGEN_IF_CONSTEXPR (!isBRowMajor) handmade_aligned_free(B_temp);
}
// Template specializations of trsmKernelL/R for float/double and inner strides of 1.
+85 -92
View File
@@ -13,8 +13,10 @@
template <bool isARowMajor = true>
EIGEN_ALWAYS_INLINE int64_t idA(int64_t i, int64_t j, int64_t LDA) {
EIGEN_IF_CONSTEXPR(isARowMajor) return i * LDA + j;
else return i + j * LDA;
EIGEN_IF_CONSTEXPR (isARowMajor)
return i * LDA + j;
else
return i + j * LDA;
}
/**
@@ -59,11 +61,11 @@ namespace unrolls {
template <int64_t N>
EIGEN_ALWAYS_INLINE auto remMask(int64_t m) {
EIGEN_IF_CONSTEXPR(N == 16) { return 0xFFFF >> (16 - m); }
else EIGEN_IF_CONSTEXPR(N == 8) {
EIGEN_IF_CONSTEXPR (N == 16) {
return 0xFFFF >> (16 - m);
} else EIGEN_IF_CONSTEXPR (N == 8) {
return 0xFF >> (8 - m);
}
else EIGEN_IF_CONSTEXPR(N == 4) {
} else EIGEN_IF_CONSTEXPR (N == 4) {
return 0x0F >> (4 - m);
}
return 0;
@@ -155,22 +157,20 @@ class trans {
constexpr int64_t counterReverse = endN - counter;
constexpr int64_t startN = counterReverse;
EIGEN_IF_CONSTEXPR(startN < EIGEN_AVX_MAX_NUM_ROW) {
EIGEN_IF_CONSTEXPR(remM) {
EIGEN_IF_CONSTEXPR (startN < EIGEN_AVX_MAX_NUM_ROW) {
EIGEN_IF_CONSTEXPR (remM) {
pstoreu<Scalar>(
C_arr + LDC * startN,
padd(ploadu<vecHalf>((const Scalar *)C_arr + LDC * startN, remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_)),
preinterpret<vecHalf>(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN]),
remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_)),
remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_));
}
else {
} else {
pstoreu<Scalar>(C_arr + LDC * startN,
padd(ploadu<vecHalf>((const Scalar *)C_arr + LDC * startN),
preinterpret<vecHalf>(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN])));
}
}
else { // This block is only needed for fp32 case
} else { // This block is only needed for fp32 case
// Reinterpret as __m512 for _mm512_shuffle_f32x4
vecFullFloat zmm2vecFullFloat = preinterpret<vecFullFloat>(
zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)]);
@@ -178,15 +178,14 @@ class trans {
zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)] =
preinterpret<vec>(_mm512_shuffle_f32x4(zmm2vecFullFloat, zmm2vecFullFloat, 0b01001110));
EIGEN_IF_CONSTEXPR(remM) {
EIGEN_IF_CONSTEXPR (remM) {
pstoreu<Scalar>(
C_arr + LDC * startN,
padd(ploadu<vecHalf>((const Scalar *)C_arr + LDC * startN, remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_)),
preinterpret<vecHalf>(
zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)])),
remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_));
}
else {
} else {
pstoreu<Scalar>(
C_arr + LDC * startN,
padd(ploadu<vecHalf>((const Scalar *)C_arr + LDC * startN),
@@ -308,16 +307,15 @@ class transB {
constexpr int64_t counterReverse = endN - counter;
constexpr int64_t startN = counterReverse;
EIGEN_IF_CONSTEXPR(remM) {
EIGEN_IF_CONSTEXPR (remM) {
ymm.packet[packetIndexOffset + startN] =
ploadu<vecHalf>((const Scalar *)&B_arr[startN * LDB], remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_));
}
else {
EIGEN_IF_CONSTEXPR(remN_ == 0) {
} else {
EIGEN_IF_CONSTEXPR (remN_ == 0) {
ymm.packet[packetIndexOffset + startN] = ploadu<vecHalf>((const Scalar *)&B_arr[startN * LDB]);
}
else ymm.packet[packetIndexOffset + startN] =
ploadu<vecHalf>((const Scalar *)&B_arr[startN * LDB], remMask<EIGEN_AVX_MAX_NUM_ROW>(remN_));
} else
ymm.packet[packetIndexOffset + startN] =
ploadu<vecHalf>((const Scalar *)&B_arr[startN * LDB], remMask<EIGEN_AVX_MAX_NUM_ROW>(remN_));
}
aux_loadB<endN, counter - 1, packetIndexOffset, remM, remN_>(B_arr, LDB, ymm, remM_);
@@ -345,11 +343,10 @@ class transB {
constexpr int64_t counterReverse = endN - counter;
constexpr int64_t startN = counterReverse;
EIGEN_IF_CONSTEXPR(remK || remM) {
EIGEN_IF_CONSTEXPR (remK || remM) {
pstoreu<Scalar>(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN],
remMask<EIGEN_AVX_MAX_NUM_ROW>(rem_));
}
else {
} else {
pstoreu<Scalar>(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN]);
}
@@ -406,10 +403,9 @@ class transB {
constexpr int64_t counterReverse = endN - counter;
constexpr int64_t startN = counterReverse;
EIGEN_IF_CONSTEXPR(toTemp) {
EIGEN_IF_CONSTEXPR (toTemp) {
transB::template storeB<EIGEN_AVX_MAX_NUM_ROW, startN, remK_ != 0, false>(&B_temp[startN], LDB_, ymm, remK_);
}
else {
} else {
transB::template storeB<std::min(EIGEN_AVX_MAX_NUM_ROW, endN), startN, false, remM>(&B_arr[0 + startN * LDB], LDB,
ymm, remM_);
}
@@ -450,8 +446,9 @@ class transB {
static EIGEN_ALWAYS_INLINE void loadBBlock(Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_,
PacketBlock<vecHalf, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> &ymm,
int64_t remM_ = 0) {
EIGEN_IF_CONSTEXPR(toTemp) { transB::template loadB<unrollN, 0, remM, 0>(&B_arr[0], LDB, ymm, remM_); }
else {
EIGEN_IF_CONSTEXPR (toTemp) {
transB::template loadB<unrollN, 0, remM, 0>(&B_arr[0], LDB, ymm, remM_);
} else {
aux_loadBBlock<unrollN, unrollN, toTemp, remM, remN_>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
}
}
@@ -501,7 +498,7 @@ class transB {
*
* For fp32 L and U1 are 1:2 so for U3/U2 cases the loads/stores need to be split up.
*/
EIGEN_IF_CONSTEXPR(unrollN == U3) {
EIGEN_IF_CONSTEXPR (unrollN == U3) {
// load LxU3 B col major, transpose LxU3 row major
constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U3);
transB::template loadBBlock<maxUBlock, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
@@ -510,7 +507,7 @@ class transB {
transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
transB::template storeBBlock<maxUBlock, toTemp, remM, 0>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
EIGEN_IF_CONSTEXPR(maxUBlock < U3) {
EIGEN_IF_CONSTEXPR (maxUBlock < U3) {
transB::template loadBBlock<maxUBlock, toTemp, remM>(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_,
ymm, remM_);
transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
@@ -519,50 +516,46 @@ class transB {
transB::template storeBBlock<maxUBlock, toTemp, remM, 0>(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_,
ymm, remM_);
}
}
else EIGEN_IF_CONSTEXPR(unrollN == U2) {
} else EIGEN_IF_CONSTEXPR (unrollN == U2) {
// load LxU2 B col major, transpose LxU2 row major
constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U2);
transB::template loadBBlock<maxUBlock, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
EIGEN_IF_CONSTEXPR(maxUBlock < U2) transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
EIGEN_IF_CONSTEXPR (maxUBlock < U2) transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
transB::template storeBBlock<maxUBlock, toTemp, remM, 0>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
EIGEN_IF_CONSTEXPR(maxUBlock < U2) {
EIGEN_IF_CONSTEXPR (maxUBlock < U2) {
transB::template loadBBlock<EIGEN_AVX_MAX_NUM_ROW, toTemp, remM>(&B_arr[maxUBlock * LDB], LDB,
&B_temp[maxUBlock], LDB_, ymm, remM_);
transB::template transposeLxL<0>(ymm);
transB::template storeBBlock<EIGEN_AVX_MAX_NUM_ROW, toTemp, remM, 0>(&B_arr[maxUBlock * LDB], LDB,
&B_temp[maxUBlock], LDB_, ymm, remM_);
}
}
else EIGEN_IF_CONSTEXPR(unrollN == U1) {
} else EIGEN_IF_CONSTEXPR (unrollN == U1) {
// load LxU1 B col major, transpose LxU1 row major
transB::template loadBBlock<U1, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
transB::template transposeLxL<0>(ymm);
EIGEN_IF_CONSTEXPR(EIGEN_AVX_MAX_NUM_ROW < U1) { transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); }
EIGEN_IF_CONSTEXPR (EIGEN_AVX_MAX_NUM_ROW < U1) {
transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
}
transB::template storeBBlock<U1, toTemp, remM, 0>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
}
else EIGEN_IF_CONSTEXPR(unrollN == 8 && U1 > 8) {
} else EIGEN_IF_CONSTEXPR (unrollN == 8 && U1 > 8) {
// load Lx4 B col major, transpose Lx4 row major
transB::template loadBBlock<8, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
transB::template transposeLxL<0>(ymm);
transB::template storeBBlock<8, toTemp, remM, 8>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
}
else EIGEN_IF_CONSTEXPR(unrollN == 4 && U1 > 4) {
} else EIGEN_IF_CONSTEXPR (unrollN == 4 && U1 > 4) {
// load Lx4 B col major, transpose Lx4 row major
transB::template loadBBlock<4, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
transB::template transposeLxL<0>(ymm);
transB::template storeBBlock<4, toTemp, remM, 4>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
}
else EIGEN_IF_CONSTEXPR(unrollN == 2) {
} else EIGEN_IF_CONSTEXPR (unrollN == 2) {
// load Lx2 B col major, transpose Lx2 row major
transB::template loadBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
transB::template transposeLxL<0>(ymm);
transB::template storeBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
}
else EIGEN_IF_CONSTEXPR(unrollN == 1) {
} else EIGEN_IF_CONSTEXPR (unrollN == 1) {
// load Lx1 B col major, transpose Lx1 row major
transB::template loadBBlock<1, toTemp, remM, 1>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
transB::template transposeLxL<0>(ymm);
@@ -614,10 +607,9 @@ class trsm {
constexpr int64_t packetIndex = startM * endK + startK;
constexpr int64_t startM_ = isFWDSolve ? startM : -startM;
const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB;
EIGEN_IF_CONSTEXPR(krem) {
EIGEN_IF_CONSTEXPR (krem) {
RHSInPacket.packet[packetIndex] = ploadu<vec>(&B_arr[rhsIndex], remMask<PacketSize>(rem));
}
else {
} else {
RHSInPacket.packet[packetIndex] = ploadu<vec>(&B_arr[rhsIndex]);
}
aux_loadRHS<isFWDSolve, endM, endK, counter - 1, krem>(B_arr, LDB, RHSInPacket, rem);
@@ -649,10 +641,9 @@ class trsm {
constexpr int64_t packetIndex = startM * endK + startK;
constexpr int64_t startM_ = isFWDSolve ? startM : -startM;
const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB;
EIGEN_IF_CONSTEXPR(krem) {
EIGEN_IF_CONSTEXPR (krem) {
pstoreu<Scalar>(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex], remMask<PacketSize>(rem));
}
else {
} else {
pstoreu<Scalar>(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex]);
}
aux_storeRHS<isFWDSolve, endM, endK, counter - 1, krem>(B_arr, LDB, RHSInPacket, rem);
@@ -711,26 +702,27 @@ class trsm {
// For each row of A, first update all corresponding RHS
constexpr int64_t packetIndex = startM * endK + startK;
EIGEN_IF_CONSTEXPR(currentM > 0) {
EIGEN_IF_CONSTEXPR (currentM > 0) {
RHSInPacket.packet[packetIndex] =
pnmadd(AInPacket.packet[startM], RHSInPacket.packet[(currentM - 1) * endK + startK],
RHSInPacket.packet[packetIndex]);
}
EIGEN_IF_CONSTEXPR(startK == endK - 1) {
EIGEN_IF_CONSTEXPR (startK == endK - 1) {
// Once all RHS for previous row of A is updated, we broadcast the next element in the column A_{i, currentM}.
EIGEN_IF_CONSTEXPR(startM == currentM && !isUnitDiag) {
EIGEN_IF_CONSTEXPR (startM == currentM && !isUnitDiag) {
// If diagonal is not unit, we broadcast reciprocals of diagonals AinPacket.packet[currentM].
// This will be used in divRHSByDiag
EIGEN_IF_CONSTEXPR(isFWDSolve)
AInPacket.packet[currentM] = pset1<vec>(Scalar(1) / A_arr[idA<isARowMajor>(currentM, currentM, LDA)]);
else AInPacket.packet[currentM] = pset1<vec>(Scalar(1) / A_arr[idA<isARowMajor>(-currentM, -currentM, LDA)]);
}
else {
EIGEN_IF_CONSTEXPR (isFWDSolve)
AInPacket.packet[currentM] = pset1<vec>(Scalar(1) / A_arr[idA<isARowMajor>(currentM, currentM, LDA)]);
else
AInPacket.packet[currentM] = pset1<vec>(Scalar(1) / A_arr[idA<isARowMajor>(-currentM, -currentM, LDA)]);
} else {
// Broadcast next off diagonal element of A
EIGEN_IF_CONSTEXPR(isFWDSolve)
AInPacket.packet[startM] = pset1<vec>(A_arr[idA<isARowMajor>(startM, currentM, LDA)]);
else AInPacket.packet[startM] = pset1<vec>(A_arr[idA<isARowMajor>(-startM, -currentM, LDA)]);
EIGEN_IF_CONSTEXPR (isFWDSolve)
AInPacket.packet[startM] = pset1<vec>(A_arr[idA<isARowMajor>(startM, currentM, LDA)]);
else
AInPacket.packet[startM] = pset1<vec>(A_arr[idA<isARowMajor>(-startM, -currentM, LDA)]);
}
}
@@ -769,8 +761,8 @@ class trsm {
// Without "if constexpr" the compiler instantiates the case <-1, numK>
// this is handled with enable_if to prevent out-of-bound warnings
// from the compiler
EIGEN_IF_CONSTEXPR(!isUnitDiag && startM > 0)
trsm::template divRHSByDiag<startM - 1, numK>(RHSInPacket, AInPacket);
EIGEN_IF_CONSTEXPR (!isUnitDiag && startM > 0)
trsm::template divRHSByDiag<startM - 1, numK>(RHSInPacket, AInPacket);
// After division, the rhs corresponding to subsequent rows of A can be partially updated
// We also broadcast the reciprocal of the next diagonal to AInPacket.packet[currentM] (if needed)
@@ -779,8 +771,8 @@ class trsm {
AInPacket);
// Handle division for the RHS corresponding to the final row of A.
EIGEN_IF_CONSTEXPR(!isUnitDiag && startM == endM - 1)
trsm::template divRHSByDiag<startM, numK>(RHSInPacket, AInPacket);
EIGEN_IF_CONSTEXPR (!isUnitDiag && startM == endM - 1)
trsm::template divRHSByDiag<startM, numK>(RHSInPacket, AInPacket);
aux_triSolveMicroKernel<isARowMajor, isFWDSolve, isUnitDiag, endM, counter - 1, numK>(A_arr, LDA, RHSInPacket,
AInPacket);
@@ -916,12 +908,13 @@ class gemm {
constexpr int64_t startM = counterReverse / (endN);
constexpr int64_t startN = counterReverse % endN;
EIGEN_IF_CONSTEXPR(rem)
zmm.packet[startN * endM + startM] =
padd(ploadu<vec>(&C_arr[(startN)*LDC + startM * PacketSize], remMask<PacketSize>(rem_)),
zmm.packet[startN * endM + startM], remMask<PacketSize>(rem_));
else zmm.packet[startN * endM + startM] =
padd(ploadu<vec>(&C_arr[(startN)*LDC + startM * PacketSize]), zmm.packet[startN * endM + startM]);
EIGEN_IF_CONSTEXPR (rem)
zmm.packet[startN * endM + startM] =
padd(ploadu<vec>(&C_arr[(startN)*LDC + startM * PacketSize], remMask<PacketSize>(rem_)),
zmm.packet[startN * endM + startM], remMask<PacketSize>(rem_));
else
zmm.packet[startN * endM + startM] =
padd(ploadu<vec>(&C_arr[(startN)*LDC + startM * PacketSize]), zmm.packet[startN * endM + startM]);
aux_updateC<endM, endN, counter - 1, rem>(C_arr, LDC, zmm, rem_);
}
@@ -949,10 +942,11 @@ class gemm {
constexpr int64_t startM = counterReverse / (endN);
constexpr int64_t startN = counterReverse % endN;
EIGEN_IF_CONSTEXPR(rem)
pstoreu<Scalar>(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM],
remMask<PacketSize>(rem_));
else pstoreu<Scalar>(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM]);
EIGEN_IF_CONSTEXPR (rem)
pstoreu<Scalar>(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM],
remMask<PacketSize>(rem_));
else
pstoreu<Scalar>(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM]);
aux_storeC<endM, endN, counter - 1, rem>(C_arr, LDC, zmm, rem_);
}
@@ -978,11 +972,12 @@ class gemm {
constexpr int64_t counterReverse = endL - counter;
constexpr int64_t startL = counterReverse;
EIGEN_IF_CONSTEXPR(rem)
zmm.packet[unrollM * unrollN + startL] =
ploadu<vec>(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize], remMask<PacketSize>(rem_));
else zmm.packet[unrollM * unrollN + startL] =
ploadu<vec>(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize]);
EIGEN_IF_CONSTEXPR (rem)
zmm.packet[unrollM * unrollN + startL] =
ploadu<vec>(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize], remMask<PacketSize>(rem_));
else
zmm.packet[unrollM * unrollN + startL] =
ploadu<vec>(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize]);
aux_startLoadB<unrollM, unrollN, endL, counter - 1, rem>(B_t, LDB, zmm, rem_);
}
@@ -1037,11 +1032,10 @@ class gemm {
constexpr int64_t counterReverse = endM - counter;
constexpr int64_t startM = counterReverse;
EIGEN_IF_CONSTEXPR(rem) {
EIGEN_IF_CONSTEXPR (rem) {
zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] =
ploadu<vec>(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize], remMask<PacketSize>(rem_));
}
else {
} else {
zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] =
ploadu<vec>(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize]);
}
@@ -1079,32 +1073,31 @@ class gemm {
constexpr int startN = (counterReverse / (endM)) % endN;
constexpr int startM = counterReverse % endM;
EIGEN_IF_CONSTEXPR(startK == 0 && startM == 0 && startN == 0) {
EIGEN_IF_CONSTEXPR (startK == 0 && startM == 0 && startN == 0) {
gemm::template startLoadB<endM, endN, numLoad, rem>(B_t, LDB, zmm, rem_);
gemm::template startBCastA<isARowMajor, endM, endN, numBCast, numLoad>(A_t, LDA, zmm);
}
{
// Interleave FMA and Bcast
EIGEN_IF_CONSTEXPR(isAdd) {
EIGEN_IF_CONSTEXPR (isAdd) {
zmm.packet[startN * endM + startM] =
pmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast],
zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]);
}
else {
} else {
zmm.packet[startN * endM + startM] =
pnmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast],
zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]);
}
// Bcast
EIGEN_IF_CONSTEXPR(startM == endM - 1 && (numBCast + startN + startK * endN < endK * endN)) {
EIGEN_IF_CONSTEXPR (startM == endM - 1 && (numBCast + startN + startK * endN < endK * endN)) {
zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast] = pload1<vec>(&A_t[idA<isARowMajor>(
(numBCast + startN + startK * endN) % endN, (numBCast + startN + startK * endN) / endN, LDA)]);
}
}
// We have updated all accumulators, time to load next set of B's
EIGEN_IF_CONSTEXPR((startN == endN - 1) && (startM == endM - 1)) {
EIGEN_IF_CONSTEXPR ((startN == endN - 1) && (startM == endM - 1)) {
gemm::template loadB<endM, endN, startK, endK, numLoad, numBCast, rem>(B_t, LDB, zmm, rem_);
}
aux_microKernel<isARowMajor, endM, endN, endK, counter - 1, numLoad, numBCast, rem>(B_t, A_t, LDB, LDA, zmm, rem_);
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+3 -4
View File
@@ -59,11 +59,10 @@ EIGEN_STRONG_INLINE HVX_Vector HVX_load_partial(const T* mem) {
HVX_Vector v0 = HVX_vmem<0>(mem);
HVX_Vector v1 = v0;
uintptr_t mem_addr = reinterpret_cast<uintptr_t>(mem);
EIGEN_IF_CONSTEXPR(Size * sizeof(T) <= Alignment) {
EIGEN_IF_CONSTEXPR (Size * sizeof(T) <= Alignment) {
// Data size less than alignment will never cross multiple aligned vectors.
v1 = v0;
}
else {
} else {
uintptr_t left_off = mem_addr & (__HVX_LENGTH__ - 1);
if (left_off + Size * sizeof(T) > __HVX_LENGTH__) {
v1 = HVX_vmem<1>(mem);
@@ -99,7 +98,7 @@ EIGEN_STRONG_INLINE void HVX_store_partial(T* mem, HVX_Vector v) {
HVX_VectorPred ql_not = Q6_Q_vsetq_R(mem_addr);
HVX_VectorPred qr = Q6_Q_vsetq2_R(right_off);
EIGEN_IF_CONSTEXPR(Size * sizeof(T) > Alignment) {
EIGEN_IF_CONSTEXPR (Size * sizeof(T) > Alignment) {
if (right_off > __HVX_LENGTH__) {
Q6_vmem_QRIV(qr, mem + __HVX_LENGTH__ / sizeof(T), value);
qr = Q6_Q_vcmp_eq_VbVb(value, value);
@@ -1144,13 +1144,14 @@ struct gebp_rhs_cols<J, MrPackets, NrCols, true> {
static EIGEN_ALWAYS_INLINE void run(GEBPTraits& traits, const RhsScalar* blB, Index rhs_offset, LhsArray& A,
RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) {
constexpr int lane = J % 4;
EIGEN_IF_CONSTEXPR(lane == 0)
traits.loadRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
else traits.updateRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
EIGEN_IF_CONSTEXPR (lane == 0)
traits.loadRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
else
traits.updateRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
EIGEN_IF_CONSTEXPR(MrPackets >= 1) traits.madd(A[0], rhs_panel, C[J + 0 * NrCols], T0, fix<lane>);
EIGEN_IF_CONSTEXPR(MrPackets >= 2) traits.madd(A[1], rhs_panel, C[J + 1 * NrCols], T0, fix<lane>);
EIGEN_IF_CONSTEXPR(MrPackets >= 3) traits.madd(A[2], rhs_panel, C[J + 2 * NrCols], T0, fix<lane>);
EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.madd(A[0], rhs_panel, C[J + 0 * NrCols], T0, fix<lane>);
EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.madd(A[1], rhs_panel, C[J + 1 * NrCols], T0, fix<lane>);
EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.madd(A[2], rhs_panel, C[J + 2 * NrCols], T0, fix<lane>);
gebp_rhs_cols<J + 1, MrPackets, NrCols>::run(traits, blB, rhs_offset, A, rhs_panel, T0, C);
}
@@ -1166,9 +1167,9 @@ struct gebp_micro_step {
RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) {
constexpr int LhsProg = GEBPTraits::LhsProgress;
EIGEN_IF_CONSTEXPR(MrPackets >= 1) traits.loadLhs(&blA[(0 + MrPackets * K) * LhsProg], A[0]);
EIGEN_IF_CONSTEXPR(MrPackets >= 2) traits.loadLhs(&blA[(1 + MrPackets * K) * LhsProg], A[1]);
EIGEN_IF_CONSTEXPR(MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]);
EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.loadLhs(&blA[(0 + MrPackets * K) * LhsProg], A[0]);
EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.loadLhs(&blA[(1 + MrPackets * K) * LhsProg], A[1]);
EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]);
gebp_rhs_cols<0, MrPackets, NrCols>::run(traits, blB, Index(NrCols * K), A, rhs_panel, T0, C);
}
@@ -1186,7 +1187,9 @@ EIGEN_ALWAYS_INLINE void gebp_neon_3p_workaround(LhsArray_& A) {
#if EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
using LhsElement = std::remove_all_extents_t<std::remove_reference_t<LhsArray_>>;
constexpr bool apply = GEBPTraits_::Vectorizable && MrPackets == 3 && std::is_same<LhsElement, FullLhsPacket_>::value;
EIGEN_IF_CONSTEXPR(apply) { __asm__("" : "+w,m"(A[0]), "+w,m"(A[1]), "+w,m"(A[2])); }
EIGEN_IF_CONSTEXPR (apply) {
__asm__("" : "+w,m"(A[0]), "+w,m"(A[1]), "+w,m"(A[2]));
}
#else
EIGEN_UNUSED_VARIABLE(A);
#endif
@@ -1204,7 +1207,7 @@ EIGEN_ALWAYS_INLINE void gebp_sse_spilling_workaround(LhsArray_& A, AccArray_& A
using LhsElement = std::remove_all_extents_t<std::remove_reference_t<LhsArray_>>;
constexpr bool apply =
GEBPTraits_::Vectorizable && MrPackets <= 2 && NrCols >= 4 && std::is_same<LhsElement, FullLhsPacket_>::value;
EIGEN_IF_CONSTEXPR(apply) {
EIGEN_IF_CONSTEXPR (apply) {
#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
using AccElement = std::decay_t<decltype(ACC[0])>;
constexpr bool pin_acc = std::is_same<AccElement, FullLhsPacket_>::value && MrPackets == 2 && NrCols == 4;
@@ -1214,7 +1217,9 @@ EIGEN_ALWAYS_INLINE void gebp_sse_spilling_workaround(LhsArray_& A, AccArray_& A
"+x"(ACC[7]));
}
#else
EIGEN_IF_CONSTEXPR(MrPackets == 2) { __asm__("" : "+x,m"(A[0]), "+x,m"(A[1])); }
EIGEN_IF_CONSTEXPR (MrPackets == 2) {
__asm__("" : "+x,m"(A[0]), "+x,m"(A[1]));
}
#endif
}
#endif
@@ -1231,7 +1236,9 @@ struct gebp_peeled_loop {
constexpr bool use_double_accum = (MrPackets == 1 && NrCols == 4);
// Prefetch for 4-col paths
EIGEN_IF_CONSTEXPR(NrCols == 4) { internal::prefetch(blB + (48 + 0)); }
EIGEN_IF_CONSTEXPR (NrCols == 4) {
internal::prefetch(blB + (48 + 0));
}
// Helper to do one step with workarounds
#define EIGEN_GEBP_DO_STEP(KVAL, ACC) \
@@ -1240,7 +1247,7 @@ struct gebp_peeled_loop {
gebp_neon_3p_workaround<MrPackets, GEBPTraits, FullLhsPacket>(A); \
gebp_sse_spilling_workaround<MrPackets, NrCols, GEBPTraits, FullLhsPacket>(A, ACC); \
/* LHS prefetch for 2pX4 and 3pX4 */ \
EIGEN_IF_CONSTEXPR((MrPackets == 2 || MrPackets == 3) && NrCols == 4) { \
EIGEN_IF_CONSTEXPR ((MrPackets == 2 || MrPackets == 3) && NrCols == 4) { \
internal::prefetch(blA + (MrPackets * KVAL + 16) * GEBPTraits::LhsProgress); \
if (EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) { \
internal::prefetch(blB + (NrCols * KVAL + 16) * GEBPTraits::RhsProgress); \
@@ -1248,23 +1255,26 @@ struct gebp_peeled_loop {
} \
} while (false)
EIGEN_IF_CONSTEXPR(use_double_accum) {
EIGEN_IF_CONSTEXPR (use_double_accum) {
EIGEN_GEBP_DO_STEP(0, C);
EIGEN_GEBP_DO_STEP(1, D);
EIGEN_GEBP_DO_STEP(2, C);
EIGEN_GEBP_DO_STEP(3, D);
EIGEN_IF_CONSTEXPR(NrCols == 4) { internal::prefetch(blB + (48 + 16)); }
EIGEN_IF_CONSTEXPR (NrCols == 4) {
internal::prefetch(blB + (48 + 16));
}
EIGEN_GEBP_DO_STEP(4, C);
EIGEN_GEBP_DO_STEP(5, D);
EIGEN_GEBP_DO_STEP(6, C);
EIGEN_GEBP_DO_STEP(7, D);
}
else {
} else {
EIGEN_GEBP_DO_STEP(0, C);
EIGEN_GEBP_DO_STEP(1, C);
EIGEN_GEBP_DO_STEP(2, C);
EIGEN_GEBP_DO_STEP(3, C);
EIGEN_IF_CONSTEXPR(NrCols == 4 && MrPackets == 2) { internal::prefetch(blB + (48 + 16)); }
EIGEN_IF_CONSTEXPR (NrCols == 4 && MrPackets == 2) {
internal::prefetch(blB + (48 + 16));
}
EIGEN_GEBP_DO_STEP(4, C);
EIGEN_GEBP_DO_STEP(5, C);
EIGEN_GEBP_DO_STEP(6, C);
@@ -1321,7 +1331,7 @@ EIGEN_ALWAYS_INLINE void gebp_micro_panel_impl(GEBPTraits& traits, const DataMap
// compiler that D[n] is always in bounds for the use_double_accum path.
alignas(AccPacketLocal) AccPacketLocal D[CSize];
#endif
EIGEN_IF_CONSTEXPR(use_double_accum) {
EIGEN_IF_CONSTEXPR (use_double_accum) {
for (int n = 0; n < NrCols; ++n) traits.initAcc(D[n]);
}
@@ -1353,7 +1363,7 @@ EIGEN_ALWAYS_INLINE void gebp_micro_panel_impl(GEBPTraits& traits, const DataMap
}
// Merge double accumulators
EIGEN_IF_CONSTEXPR(use_double_accum) {
EIGEN_IF_CONSTEXPR (use_double_accum) {
for (int n = 0; n < NrCols; ++n) C[n] = padd(C[n], D[n]);
}
@@ -1456,7 +1466,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
}
//---------- Process 3 * LhsProgress rows at once ----------
EIGEN_IF_CONSTEXPR(mr >= 3 * Traits::LhsProgress) {
EIGEN_IF_CONSTEXPR (mr >= 3 * Traits::LhsProgress) {
const Index rhs_block = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar);
const Index lhs_strip = depth * sizeof(LhsScalar) * 3 * LhsProgress;
const Index lhs_avail = (lhs_budget > rhs_block) ? (lhs_budget - rhs_block) : 0;
@@ -1465,7 +1475,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
: (3 * LhsProgress) * std::max<Index>(1, lhs_avail / lhs_strip);
for (Index i1 = 0; i1 < peeled_mc3; i1 += actual_panel_rows) {
const Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc3);
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
for (Index i = i1; i < actual_panel_end; i += 3 * LhsProgress) {
micro_panel(fix<3>, fix<8>, traits, i, j2);
@@ -1486,7 +1496,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
}
//---------- Process 2 * LhsProgress rows at once ----------
EIGEN_IF_CONSTEXPR(mr >= 2 * Traits::LhsProgress) {
EIGEN_IF_CONSTEXPR (mr >= 2 * Traits::LhsProgress) {
const Index rhs_block2 = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar);
const Index lhs_strip2 = depth * sizeof(LhsScalar) * 2 * LhsProgress;
const Index lhs_avail2 = (lhs_budget > rhs_block2) ? (lhs_budget - rhs_block2) : 0;
@@ -1496,7 +1506,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
: (2 * LhsProgress) * std::max<Index>(1, lhs_avail2 / lhs_strip2);
for (Index i1 = peeled_mc3; i1 < peeled_mc2; i1 += actual_panel_rows) {
Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc2);
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
for (Index i = i1; i < actual_panel_end; i += 2 * LhsProgress) {
micro_panel(fix<2>, fix<8>, traits, i, j2);
@@ -1517,9 +1527,9 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
}
//---------- Process 1 * LhsProgress rows at once ----------
EIGEN_IF_CONSTEXPR(mr >= 1 * Traits::LhsProgress) {
EIGEN_IF_CONSTEXPR (mr >= 1 * Traits::LhsProgress) {
for (Index i = peeled_mc2; i < peeled_mc1; i += LhsProgress) {
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
micro_panel(fix<1>, fix<8>, traits, i, j2);
}
@@ -1534,10 +1544,10 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
}
//---------- Process LhsProgressHalf rows at once ----------
EIGEN_IF_CONSTEXPR((LhsProgressHalf < LhsProgress) && mr >= LhsProgressHalf) {
EIGEN_IF_CONSTEXPR ((LhsProgressHalf < LhsProgress) && mr >= LhsProgressHalf) {
HalfTraits half_traits;
for (Index i = peeled_mc1; i < peeled_mc_half; i += LhsProgressHalf) {
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
gebp_micro_panel_impl<1, 8, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
@@ -1558,10 +1568,10 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
}
//---------- Process LhsProgressQuarter rows at once ----------
EIGEN_IF_CONSTEXPR((LhsProgressQuarter < LhsProgressHalf) && mr >= LhsProgressQuarter) {
EIGEN_IF_CONSTEXPR ((LhsProgressQuarter < LhsProgressHalf) && mr >= LhsProgressQuarter) {
QuarterTraits quarter_traits;
for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += LhsProgressQuarter) {
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
gebp_micro_panel_impl<1, 8, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
@@ -1583,7 +1593,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
//---------- Process remaining rows, 1 at once ----------
if (peeled_mc_quarter < rows) {
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
// loop on each panel of the rhs
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
// loop on each row of the lhs (1*LhsProgress x depth)
@@ -1862,9 +1872,9 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
Index i = 0;
// Pack 3 packets
EIGEN_IF_CONSTEXPR(Pack1 >= 3 * PacketSize) {
EIGEN_IF_CONSTEXPR (Pack1 >= 3 * PacketSize) {
for (; i < peeled_mc3; i += 3 * PacketSize) {
EIGEN_IF_CONSTEXPR(PanelMode) count += (3 * PacketSize) * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += (3 * PacketSize) * offset;
for (Index k = 0; k < depth; k++) {
Packet A, B, C;
@@ -1878,13 +1888,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
pstore(blockA + count, cj.pconj(C));
count += PacketSize;
}
EIGEN_IF_CONSTEXPR(PanelMode) count += (3 * PacketSize) * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += (3 * PacketSize) * (stride - offset - depth);
}
}
// Pack 2 packets
EIGEN_IF_CONSTEXPR(Pack1 >= 2 * PacketSize) {
EIGEN_IF_CONSTEXPR (Pack1 >= 2 * PacketSize) {
for (; i < peeled_mc2; i += 2 * PacketSize) {
EIGEN_IF_CONSTEXPR(PanelMode) count += (2 * PacketSize) * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += (2 * PacketSize) * offset;
for (Index k = 0; k < depth; k++) {
Packet A, B;
@@ -1895,13 +1905,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
pstore(blockA + count, cj.pconj(B));
count += PacketSize;
}
EIGEN_IF_CONSTEXPR(PanelMode) count += (2 * PacketSize) * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += (2 * PacketSize) * (stride - offset - depth);
}
}
// Pack 1 packets
EIGEN_IF_CONSTEXPR(Pack1 >= 1 * PacketSize) {
EIGEN_IF_CONSTEXPR (Pack1 >= 1 * PacketSize) {
for (; i < peeled_mc1; i += 1 * PacketSize) {
EIGEN_IF_CONSTEXPR(PanelMode) count += (1 * PacketSize) * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += (1 * PacketSize) * offset;
for (Index k = 0; k < depth; k++) {
Packet A;
@@ -1909,13 +1919,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
pstore(blockA + count, cj.pconj(A));
count += PacketSize;
}
EIGEN_IF_CONSTEXPR(PanelMode) count += (1 * PacketSize) * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += (1 * PacketSize) * (stride - offset - depth);
}
}
// Pack half packets
EIGEN_IF_CONSTEXPR(HasHalf && Pack1 >= HalfPacketSize) {
EIGEN_IF_CONSTEXPR (HasHalf && Pack1 >= HalfPacketSize) {
for (; i < peeled_mc_half; i += HalfPacketSize) {
EIGEN_IF_CONSTEXPR(PanelMode) count += (HalfPacketSize)*offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += (HalfPacketSize)*offset;
for (Index k = 0; k < depth; k++) {
HalfPacket A;
@@ -1923,13 +1933,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
pstoreu(blockA + count, cj.pconj(A));
count += HalfPacketSize;
}
EIGEN_IF_CONSTEXPR(PanelMode) count += (HalfPacketSize) * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += (HalfPacketSize) * (stride - offset - depth);
}
}
// Pack quarter packets
EIGEN_IF_CONSTEXPR(HasQuarter && Pack1 >= QuarterPacketSize) {
EIGEN_IF_CONSTEXPR (HasQuarter && Pack1 >= QuarterPacketSize) {
for (; i < peeled_mc_quarter; i += QuarterPacketSize) {
EIGEN_IF_CONSTEXPR(PanelMode) count += (QuarterPacketSize)*offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += (QuarterPacketSize)*offset;
for (Index k = 0; k < depth; k++) {
QuarterPacket A;
@@ -1937,7 +1947,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
pstoreu(blockA + count, cj.pconj(A));
count += QuarterPacketSize;
}
EIGEN_IF_CONSTEXPR(PanelMode) count += (QuarterPacketSize) * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += (QuarterPacketSize) * (stride - offset - depth);
}
}
// Pack2 may be *smaller* than PacketSize—that happens for
@@ -1953,23 +1963,23 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
// that case we use exactly Pack2 rows per group so the kernel's main
// loop (which reads Pack2 = LhsProgress values via ploaddup) can
// handle them; remaining rows fall through to the scalar loop below.
EIGEN_IF_CONSTEXPR(Pack2 < PacketSize && Pack2 > 1) {
EIGEN_IF_CONSTEXPR (Pack2 < PacketSize && Pack2 > 1) {
const Index pack2_progress = (HasHalf || HasQuarter) ? last_lhs_progress : Pack2;
const Index peeled = (HasHalf || HasQuarter) ? peeled_mc0 : (rows / Pack2) * Pack2;
for (; i < peeled; i += pack2_progress) {
EIGEN_IF_CONSTEXPR(PanelMode) count += pack2_progress * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * offset;
for (Index k = 0; k < depth; k++)
for (Index w = 0; w < pack2_progress; w++) blockA[count++] = cj(lhs(i + w, k));
EIGEN_IF_CONSTEXPR(PanelMode) count += pack2_progress * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * (stride - offset - depth);
}
}
// Pack scalars
for (; i < rows; i++) {
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k));
EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
}
}
@@ -2012,7 +2022,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
Index peeled_mc = gone_last ? Pack2 > 1 ? (rows / pack) * pack : 0 : i + (remaining_rows / pack) * pack;
Index starting_pos = i;
for (; i < peeled_mc; i += pack) {
EIGEN_IF_CONSTEXPR(PanelMode) count += pack * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += pack * offset;
Index k = 0;
if (pack >= psize && psize >= QuarterPacketSize) {
@@ -2058,7 +2068,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
for (; w < pack; ++w) blockA[count++] = cj(lhs(i + w, k));
}
EIGEN_IF_CONSTEXPR(PanelMode) count += pack * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += pack * (stride - offset - depth);
}
pack -= psize;
@@ -2084,7 +2094,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
// that case we use exactly Pack2 rows per group so the kernel's main
// loop (which reads Pack2 = LhsProgress values via ploaddup) can
// handle them; remaining rows fall through to the scalar loop below.
EIGEN_IF_CONSTEXPR(Pack2 < PacketSize) {
EIGEN_IF_CONSTEXPR (Pack2 < PacketSize) {
if (!gone_last) {
gone_last = true;
psize = pack = (HasHalf || HasQuarter) ? (left & ~1) : Pack2;
@@ -2094,9 +2104,9 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
}
for (; i < rows; i++) {
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k));
EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
}
}
@@ -2129,10 +2139,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
Index count = 0;
const Index peeled_k = (depth / PacketSize) * PacketSize;
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
// skip what we have before
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
@@ -2142,10 +2152,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
const LinearMapper dm6 = rhs.getLinearMapper(0, j2 + 6);
const LinearMapper dm7 = rhs.getLinearMapper(0, j2 + 7);
Index k = 0;
EIGEN_IF_CONSTEXPR(PacketSize % 2 == 0 && PacketSize <= 8) // 2 4 8
EIGEN_IF_CONSTEXPR (PacketSize % 2 == 0 && PacketSize <= 8) // 2 4 8
{
for (; k < peeled_k; k += PacketSize) {
EIGEN_IF_CONSTEXPR(PacketSize == 2) {
EIGEN_IF_CONSTEXPR (PacketSize == 2) {
PacketBlock<Packet, PacketSize == 2 ? 2 : PacketSize> kernel0, kernel1, kernel2, kernel3;
kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
kernel0.packet[1 % PacketSize] = dm1.template loadPacket<Packet>(k);
@@ -2170,8 +2180,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel2.packet[1 % PacketSize]));
pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel3.packet[1 % PacketSize]));
count += 8 * PacketSize;
}
else EIGEN_IF_CONSTEXPR(PacketSize == 4) {
} else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
PacketBlock<Packet, PacketSize == 4 ? 4 : PacketSize> kernel0, kernel1;
kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
@@ -2194,8 +2203,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel0.packet[3 % PacketSize]));
pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel1.packet[3 % PacketSize]));
count += 8 * PacketSize;
}
else EIGEN_IF_CONSTEXPR(PacketSize == 8) {
} else EIGEN_IF_CONSTEXPR (PacketSize == 8) {
PacketBlock<Packet, PacketSize == 8 ? 8 : PacketSize> kernel0;
kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
@@ -2233,28 +2241,28 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
count += 8;
}
// skip what we have after
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
}
}
EIGEN_IF_CONSTEXPR(nr >= 4) {
EIGEN_IF_CONSTEXPR (nr >= 4) {
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
// skip what we have before
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3);
Index k = 0;
EIGEN_IF_CONSTEXPR((PacketSize % 4) == 0 || PacketSize == 2) {
EIGEN_IF_CONSTEXPR ((PacketSize % 4) == 0 || PacketSize == 2) {
for (; k < peeled_k; k += PacketSize) {
PacketBlock<Packet, 4> kernel;
kernel.packet[0] = dm0.template loadPacket<Packet>(k);
kernel.packet[1] = dm1.template loadPacket<Packet>(k);
kernel.packet[2] = dm2.template loadPacket<Packet>(k);
kernel.packet[3] = dm3.template loadPacket<Packet>(k);
EIGEN_IF_CONSTEXPR(PacketSize == 2) {
EIGEN_IF_CONSTEXPR (PacketSize == 2) {
// For PacketSize==2 we cannot ptranspose 4 packets directly; compose two
// 2-packet transposes and re-interleave so the 4 stores produce the
// packed-rhs layout (each store writing one half-row of the panel).
@@ -2270,8 +2278,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
kernel.packet[1] = tmp23.packet[0];
kernel.packet[2] = tmp01.packet[1];
kernel.packet[3] = tmp23.packet[1];
}
else {
} else {
ptranspose(kernel);
}
pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0]));
@@ -2289,19 +2296,19 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
count += 4;
}
// skip what we have after
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
}
}
// copy the remaining columns one at a time (nr==1)
for (Index j2 = packet_cols4; j2 < cols; ++j2) {
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
const LinearMapper dm0 = rhs.getLinearMapper(0, j2);
for (Index k = 0; k < depth; k++) {
blockB[count] = cj(dm0(k));
count += 1;
}
EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
}
}
@@ -2330,24 +2337,22 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, nr, RowMajor, Conjugate, PanelMo
Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
Index count = 0;
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
// skip what we have before
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
for (Index k = 0; k < depth; k++) {
EIGEN_IF_CONSTEXPR(PacketSize == 8) {
EIGEN_IF_CONSTEXPR (PacketSize == 8) {
Packet A = rhs.template loadPacket<Packet>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
count += PacketSize;
}
else EIGEN_IF_CONSTEXPR(PacketSize == 4) {
} else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
Packet A = rhs.template loadPacket<Packet>(k, j2);
Packet B = rhs.template loadPacket<Packet>(k, j2 + 4);
pstoreu(blockB + count, cj.pconj(A));
pstoreu(blockB + count + PacketSize, cj.pconj(B));
count += 2 * PacketSize;
}
else {
} else {
const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
blockB[count + 0] = cj(dm0(0));
blockB[count + 1] = cj(dm0(1));
@@ -2361,31 +2366,28 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, nr, RowMajor, Conjugate, PanelMo
}
}
// skip what we have after
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
}
}
EIGEN_IF_CONSTEXPR(nr >= 4) {
EIGEN_IF_CONSTEXPR (nr >= 4) {
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
// skip what we have before
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
for (Index k = 0; k < depth; k++) {
EIGEN_IF_CONSTEXPR(PacketSize == 4) {
EIGEN_IF_CONSTEXPR (PacketSize == 4) {
Packet A = rhs.template loadPacket<Packet>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
count += PacketSize;
}
else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 4) {
} else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 4) {
HalfPacket A = rhs.template loadPacket<HalfPacket>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
count += HalfPacketSize;
}
else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 4) {
} else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 4) {
QuarterPacket A = rhs.template loadPacket<QuarterPacket>(k, j2);
pstoreu(blockB + count, cj.pconj(A));
count += QuarterPacketSize;
}
else {
} else {
const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
blockB[count + 0] = cj(dm0(0));
blockB[count + 1] = cj(dm0(1));
@@ -2395,17 +2397,17 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, nr, RowMajor, Conjugate, PanelMo
}
}
// skip what we have after
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * (stride - offset - depth);
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
}
}
// copy the remaining columns one at a time (nr==1)
for (Index j2 = packet_cols4; j2 < cols; ++j2) {
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
for (Index k = 0; k < depth; k++) {
blockB[count] = cj(rhs(k, j2));
count += 1;
}
EIGEN_IF_CONSTEXPR(PanelMode) count += stride - offset - depth;
EIGEN_IF_CONSTEXPR (PanelMode) count += stride - offset - depth;
}
}
};
@@ -121,7 +121,7 @@ struct general_matrix_matrix_triangular_product<Index, LhsScalar, LhsStorageOrde
// 1 - before the diagonal => processed with gebp or skipped
// 2 - the actual_mc x actual_mc symmetric block => processed with a special kernel
// 3 - after the diagonal => processed with gebp or skipped
EIGEN_IF_CONSTEXPR(UpLo == Lower) {
EIGEN_IF_CONSTEXPR (UpLo == Lower) {
gebp(res.getSubMapper(i2, 0), blockA, blockB, actual_mc, actual_kc, (std::min)(size, i2), alpha, -1, -1, 0,
0);
}
@@ -129,7 +129,7 @@ struct general_matrix_matrix_triangular_product<Index, LhsScalar, LhsStorageOrde
sybb(res_ + resStride * i2 + resIncr * i2, resIncr, resStride, blockA, blockB + actual_kc * i2, actual_mc,
actual_kc, alpha);
EIGEN_IF_CONSTEXPR(UpLo == Upper) {
EIGEN_IF_CONSTEXPR (UpLo == Upper) {
Index j2 = i2 + actual_mc;
gebp(res.getSubMapper(i2, j2), blockA, blockB + actual_kc * j2, actual_mc, actual_kc,
(std::max)(Index(0), size - j2), alpha, -1, -1, 0, 0);
@@ -171,7 +171,7 @@ struct tribb_kernel {
Index actualBlockSize = std::min<Index>(BlockSize, size - j);
const RhsScalar* actual_b = blockB + j * depth;
EIGEN_IF_CONSTEXPR(UpLo == Upper) {
EIGEN_IF_CONSTEXPR (UpLo == Upper) {
gebp_kernel1(res.getSubMapper(0, j), blockA, actual_b, j, depth, actualBlockSize, alpha, -1, -1, 0, 0);
}
@@ -191,7 +191,7 @@ struct tribb_kernel {
}
}
EIGEN_IF_CONSTEXPR(UpLo == Lower) {
EIGEN_IF_CONSTEXPR (UpLo == Lower) {
Index i = j + actualBlockSize;
gebp_kernel1(res.getSubMapper(i, j), blockA + depth * i, actual_b, size - i, depth, actualBlockSize, alpha, -1,
-1, 0, 0);
@@ -240,7 +240,7 @@ struct general_product_to_triangular_selector<MatrixType, ProductType, UpLo, tru
ei_declare_aligned_stack_constructed_variable(
Scalar, actualLhsPtr, actualLhs.size(),
(UseLhsDirectly ? const_cast<Scalar*>(actualLhs.data()) : static_lhs.data()));
EIGEN_IF_CONSTEXPR(!UseLhsDirectly) {
EIGEN_IF_CONSTEXPR (!UseLhsDirectly) {
Map<typename ActualLhs_::PlainObject>(actualLhsPtr, actualLhs.size()) = actualLhs;
}
@@ -249,7 +249,7 @@ struct general_product_to_triangular_selector<MatrixType, ProductType, UpLo, tru
ei_declare_aligned_stack_constructed_variable(
Scalar, actualRhsPtr, actualRhs.size(),
(UseRhsDirectly ? const_cast<Scalar*>(actualRhs.data()) : static_rhs.data()));
EIGEN_IF_CONSTEXPR(!UseRhsDirectly) {
EIGEN_IF_CONSTEXPR (!UseRhsDirectly) {
Map<typename ActualRhs_::PlainObject>(actualRhsPtr, actualRhs.size()) = actualRhs;
}
@@ -255,7 +255,7 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLh
process_rows<1>(i, j2, jend, lhs, rhs, res, palpha, pcj);
i += ResPacketSize;
}
EIGEN_IF_CONSTEXPR(HasHalf) {
EIGEN_IF_CONSTEXPR (HasHalf) {
if (i < n_half) {
ResPacketHalf c0 = pzero(ResPacketHalf{});
for (Index j = j2; j < jend; j += 1) {
@@ -267,7 +267,7 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLh
i += ResPacketSizeHalf;
}
}
EIGEN_IF_CONSTEXPR(HasQuarter) {
EIGEN_IF_CONSTEXPR (HasQuarter) {
if (i < n_quarter) {
ResPacketQuarter c0 = pzero(ResPacketQuarter{});
for (Index j = j2; j < jend; j += 1) {
@@ -511,14 +511,14 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLh
c0 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i, j), b0, c0);
}
ResScalar cc0 = predux(c0);
EIGEN_IF_CONSTEXPR(HasHalf) {
EIGEN_IF_CONSTEXPR (HasHalf) {
for (Index j = fullColBlockEnd; j < halfColBlockEnd; j += LhsPacketSizeHalf) {
RhsPacketHalf b0 = rhs.template load<RhsPacketHalf, Unaligned>(j, 0);
c0_h = pcj_half.pmadd(lhs.template load<LhsPacketHalf, LhsAlignment>(i, j), b0, c0_h);
}
cc0 += predux(c0_h);
}
EIGEN_IF_CONSTEXPR(HasQuarter) {
EIGEN_IF_CONSTEXPR (HasQuarter) {
for (Index j = halfColBlockEnd; j < quarterColBlockEnd; j += LhsPacketSizeQuarter) {
RhsPacketQuarter b0 = rhs.template load<RhsPacketQuarter, Unaligned>(j, 0);
c0_q = pcj_quarter.pmadd(lhs.template load<LhsPacketQuarter, LhsAlignment>(i, j), b0, c0_q);
@@ -631,7 +631,7 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLh
using Unroll = gemv_small_cols_unroller<N - 1, N>;
ResScalar cc[N] = {};
EIGEN_IF_CONSTEXPR(HasHalf) {
EIGEN_IF_CONSTEXPR (HasHalf) {
ResPacketHalf h[N];
Unroll::init_zero(h);
for (Index j = 0; j < halfColBlockEnd; j += LhsPacketSizeHalf) {
@@ -640,7 +640,7 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLh
}
Unroll::predux_accum(cc, h);
}
EIGEN_IF_CONSTEXPR(HasQuarter) {
EIGEN_IF_CONSTEXPR (HasQuarter) {
ResPacketQuarter q[N];
Unroll::init_zero(q);
for (Index j = halfColBlockEnd; j < quarterColBlockEnd; j += LhsPacketSizeQuarter) {
@@ -69,22 +69,22 @@ struct symm_pack_lhs {
? peeled_mc_half + ((rows - peeled_mc_half) / (QuarterPacketSize)) * (QuarterPacketSize)
: 0;
EIGEN_IF_CONSTEXPR(Pack1 >= 3 * PacketSize)
for (Index i = 0; i < peeled_mc3; i += 3 * PacketSize) pack<3 * PacketSize>(blockA, lhs, cols, i, count);
EIGEN_IF_CONSTEXPR (Pack1 >= 3 * PacketSize)
for (Index i = 0; i < peeled_mc3; i += 3 * PacketSize) pack<3 * PacketSize>(blockA, lhs, cols, i, count);
EIGEN_IF_CONSTEXPR(Pack1 >= 2 * PacketSize)
for (Index i = peeled_mc3; i < peeled_mc2; i += 2 * PacketSize) pack<2 * PacketSize>(blockA, lhs, cols, i, count);
EIGEN_IF_CONSTEXPR (Pack1 >= 2 * PacketSize)
for (Index i = peeled_mc3; i < peeled_mc2; i += 2 * PacketSize) pack<2 * PacketSize>(blockA, lhs, cols, i, count);
EIGEN_IF_CONSTEXPR(Pack1 >= 1 * PacketSize)
for (Index i = peeled_mc2; i < peeled_mc1; i += 1 * PacketSize) pack<1 * PacketSize>(blockA, lhs, cols, i, count);
EIGEN_IF_CONSTEXPR (Pack1 >= 1 * PacketSize)
for (Index i = peeled_mc2; i < peeled_mc1; i += 1 * PacketSize) pack<1 * PacketSize>(blockA, lhs, cols, i, count);
EIGEN_IF_CONSTEXPR(HasHalf && Pack1 >= HalfPacketSize)
for (Index i = peeled_mc1; i < peeled_mc_half; i += HalfPacketSize)
pack<HalfPacketSize>(blockA, lhs, cols, i, count);
EIGEN_IF_CONSTEXPR (HasHalf && Pack1 >= HalfPacketSize)
for (Index i = peeled_mc1; i < peeled_mc_half; i += HalfPacketSize)
pack<HalfPacketSize>(blockA, lhs, cols, i, count);
EIGEN_IF_CONSTEXPR(HasQuarter && Pack1 >= QuarterPacketSize)
for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += QuarterPacketSize)
pack<QuarterPacketSize>(blockA, lhs, cols, i, count);
EIGEN_IF_CONSTEXPR (HasQuarter && Pack1 >= QuarterPacketSize)
for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += QuarterPacketSize)
pack<QuarterPacketSize>(blockA, lhs, cols, i, count);
// do the same with mr==1
for (Index i = peeled_mc_quarter; i < rows; i++) {
@@ -112,11 +112,11 @@ struct symm_pack_rhs {
for (Index k = k2; k < end_k; k++) {
blockB[count + 0] = rhs(k, j2 + 0);
blockB[count + 1] = rhs(k, j2 + 1);
EIGEN_IF_CONSTEXPR(nr >= 4) {
EIGEN_IF_CONSTEXPR (nr >= 4) {
blockB[count + 2] = rhs(k, j2 + 2);
blockB[count + 3] = rhs(k, j2 + 3);
}
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
blockB[count + 4] = rhs(k, j2 + 4);
blockB[count + 5] = rhs(k, j2 + 5);
blockB[count + 6] = rhs(k, j2 + 6);
@@ -128,7 +128,7 @@ struct symm_pack_rhs {
// second part: diagonal block
Index end8 = nr >= 8 ? (std::min)(k2 + rows, packet_cols8) : k2;
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = k2; j2 < end8; j2 += 8) {
// again we can split vertically in three different parts (transpose, symmetric, normal)
// transpose
@@ -170,7 +170,7 @@ struct symm_pack_rhs {
}
}
}
EIGEN_IF_CONSTEXPR(nr >= 4) {
EIGEN_IF_CONSTEXPR (nr >= 4) {
for (Index j2 = end8; j2 < (std::min)(k2 + rows, packet_cols4); j2 += 4) {
// again we can split vertically in three different parts (transpose, symmetric, normal)
// transpose
@@ -206,7 +206,7 @@ struct symm_pack_rhs {
}
// third part: transposed
EIGEN_IF_CONSTEXPR(nr >= 8) {
EIGEN_IF_CONSTEXPR (nr >= 8) {
for (Index j2 = k2 + rows; j2 < packet_cols8; j2 += 8) {
for (Index k = k2; k < end_k; k++) {
blockB[count + 0] = numext::conj(rhs(j2 + 0, k));
@@ -221,7 +221,7 @@ struct symm_pack_rhs {
}
}
}
EIGEN_IF_CONSTEXPR(nr >= 4) {
EIGEN_IF_CONSTEXPR (nr >= 4) {
for (Index j2 = (std::max)(packet_cols8, k2 + rows); j2 < packet_cols4; j2 += 4) {
for (Index k = k2; k < end_k; k++) {
blockB[count + 0] = numext::conj(rhs(j2 + 0, k));
@@ -57,11 +57,10 @@ struct selfadjoint_matrix_vector_product_symv
Specialized> { \
static void run(Index size, const Scalar* lhs, Index lhsStride, const Scalar* _rhs, Scalar* res, Scalar alpha) { \
enum { IsColMajor = StorageOrder == ColMajor }; \
EIGEN_IF_CONSTEXPR(IsColMajor == ConjugateLhs) { \
EIGEN_IF_CONSTEXPR (IsColMajor == ConjugateLhs) { \
selfadjoint_matrix_vector_product<Scalar, Index, StorageOrder, UpLo, ConjugateLhs, ConjugateRhs, \
BuiltIn>::run(size, lhs, lhsStride, _rhs, res, alpha); \
} \
else { \
} else { \
selfadjoint_matrix_vector_product_symv<Scalar, Index, StorageOrder, UpLo, ConjugateLhs, ConjugateRhs>::run( \
size, lhs, lhsStride, _rhs, res, alpha); \
} \
+3 -4
View File
@@ -40,7 +40,7 @@ struct selfadjoint_rank1_update<Scalar, Index, ColMajor, UpLo, ConjLhs, ConjRhs>
Packet ps0 = internal::pset1<Packet>(s0);
Packet ps1 = internal::pset1<Packet>(s1);
EIGEN_IF_CONSTEXPR(UpLo == Lower) {
EIGEN_IF_CONSTEXPR (UpLo == Lower) {
Scalar* EIGEN_RESTRICT col0 = mat + stride * j + j;
Scalar* EIGEN_RESTRICT col1 = mat + stride * (j + 1) + (j + 1);
@@ -71,8 +71,7 @@ struct selfadjoint_rank1_update<Scalar, Index, ColMajor, UpLo, ConjLhs, ConjRhs>
d0[k] += s0 * cx;
d1[k] += s1 * cx;
}
}
else {
} else {
// UpLo == Upper
Scalar* EIGEN_RESTRICT col0 = mat + stride * j;
Scalar* EIGEN_RESTRICT col1 = mat + stride * (j + 1);
@@ -162,7 +161,7 @@ struct selfadjoint_product_selector<MatrixType, OtherType, UpLo, true> {
Scalar, actualOtherPtr, other.size(),
(UseOtherDirectly ? const_cast<Scalar*>(actualOther.data()) : static_other.data()));
EIGEN_IF_CONSTEXPR(!UseOtherDirectly) {
EIGEN_IF_CONSTEXPR (!UseOtherDirectly) {
Map<typename ActualOtherType_::PlainObject>(actualOtherPtr, actualOther.size()) = actualOther;
}
@@ -220,7 +220,9 @@ EIGEN_DEVICE_FUNC SelfAdjointView<MatrixType, UpLo>& SelfAdjointView<MatrixType,
Scalar actualAlpha = alpha * UBlasTraits::extractScalarFactor(u.derived()) *
numext::conj(VBlasTraits::extractScalarFactor(v.derived()));
EIGEN_IF_CONSTEXPR(IsRowMajor) { actualAlpha = numext::conj(actualAlpha); }
EIGEN_IF_CONSTEXPR (IsRowMajor) {
actualAlpha = numext::conj(actualAlpha);
}
const Index size = u.size();
@@ -229,9 +231,10 @@ EIGEN_DEVICE_FUNC SelfAdjointView<MatrixType, UpLo>& SelfAdjointView<MatrixType,
static_u;
ei_declare_aligned_stack_constructed_variable(Scalar, uPtr, size,
(UseUDirectly ? const_cast<Scalar*>(actualU.data()) : static_u.data()));
EIGEN_IF_CONSTEXPR(!UseUDirectly) {
EIGEN_IF_CONSTEXPR(NeedConjU) { Map<typename ActualUType_::PlainObject>(uPtr, size) = actualU.conjugate(); }
else {
EIGEN_IF_CONSTEXPR (!UseUDirectly) {
EIGEN_IF_CONSTEXPR (NeedConjU) {
Map<typename ActualUType_::PlainObject>(uPtr, size) = actualU.conjugate();
} else {
Map<typename ActualUType_::PlainObject>(uPtr, size) = actualU;
}
}
@@ -241,9 +244,10 @@ EIGEN_DEVICE_FUNC SelfAdjointView<MatrixType, UpLo>& SelfAdjointView<MatrixType,
static_v;
ei_declare_aligned_stack_constructed_variable(Scalar, vPtr, size,
(UseVDirectly ? const_cast<Scalar*>(actualV.data()) : static_v.data()));
EIGEN_IF_CONSTEXPR(!UseVDirectly) {
EIGEN_IF_CONSTEXPR(NeedConjV) { Map<typename ActualVType_::PlainObject>(vPtr, size) = actualV.conjugate(); }
else {
EIGEN_IF_CONSTEXPR (!UseVDirectly) {
EIGEN_IF_CONSTEXPR (NeedConjV) {
Map<typename ActualVType_::PlainObject>(vPtr, size) = actualV.conjugate();
} else {
Map<typename ActualVType_::PlainObject>(vPtr, size) = actualV;
}
}
@@ -92,9 +92,10 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix<
Matrix<Scalar, SmallPanelWidth, SmallPanelWidth, LhsStorageOrder> triangularBuffer;
triangularBuffer.setZero();
EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag)
triangularBuffer.diagonal().setZero();
else triangularBuffer.diagonal().setOnes();
EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag)
triangularBuffer.diagonal().setZero();
else
triangularBuffer.diagonal().setOnes();
gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing,
@@ -217,9 +218,10 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix<
Matrix<Scalar, SmallPanelWidth, SmallPanelWidth, RhsStorageOrder> triangularBuffer;
triangularBuffer.setZero();
EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag)
triangularBuffer.diagonal().setZero();
else triangularBuffer.diagonal().setOnes();
EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag)
triangularBuffer.diagonal().setZero();
else
triangularBuffer.diagonal().setOnes();
gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing,
@@ -233,7 +235,7 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix<
Index actual_k2 = IsLower ? k2 : k2 - actual_kc;
// align blocks with the end of the triangular part for trapezoidal rhs
EIGEN_IF_CONSTEXPR(IsLower) {
EIGEN_IF_CONSTEXPR (IsLower) {
if ((k2 < cols) && (actual_k2 + actual_kc > cols)) {
actual_kc = cols - k2;
k2 = actual_k2 + actual_kc - kc;
@@ -355,7 +357,7 @@ struct triangular_product_impl<Mode, LhsIsTriangular, Lhs, false, Rhs, false> {
actualAlpha, blocking);
// Apply correction if the diagonal is unit and a scalar factor was nested:
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) {
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) {
if (LhsIsTriangular && !numext::is_exactly_one(lhs_alpha)) {
Index diagSize = (std::min)(lhs.rows(), lhs.cols());
dst.topRows(diagSize) -= ((lhs_alpha - LhsScalar(1)) * a_rhs).topRows(diagSize);
@@ -55,7 +55,7 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
// Process the triangular panel using raw pointer operations with 2-column batching
// to eliminate expression template overhead and share result loads/stores.
EIGEN_IF_CONSTEXPR(IsLower) {
EIGEN_IF_CONSTEXPR (IsLower) {
Index k = 0;
for (; k + 1 < actualPanelWidth; k += 2) {
Index i0 = pi + k;
@@ -66,18 +66,18 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
const LhsScalar* EIGEN_RESTRICT c1 = lhs_ + i1 * lhsStride;
// Diagonal of column 0
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) res_[i0] += s0 * cjl(c0[i0]);
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) res_[i0] += s0 * cjl(c0[i0]);
// Row i1: contribution from column 0 + diagonal of column 1
{
ResScalar r1 = s0 * cjl(c0[i1]);
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) r1 += s1 * cjl(c1[i1]);
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) r1 += s1 * cjl(c1[i1]);
res_[i1] += r1;
}
// Shared rows where both columns contribute
Index panelEnd = pi + actualPanelWidth;
for (Index j = i1 + 1; j < panelEnd; ++j) res_[j] += s0 * cjl(c0[j]) + s1 * cjl(c1[j]);
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
res_[i0] += s0;
res_[i1] += s1;
}
@@ -86,11 +86,10 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
Index i = pi + k;
ResScalar s = alpha * cjr(rhs_[i * rhsIncr]);
const LhsScalar* EIGEN_RESTRICT c = lhs_ + i * lhsStride;
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) res_[i] += s * cjl(c[i]);
EIGEN_IF_CONSTEXPR(HasUnitDiag) res_[i] += s;
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) res_[i] += s * cjl(c[i]);
EIGEN_IF_CONSTEXPR (HasUnitDiag) res_[i] += s;
}
}
else {
} else {
// Upper triangular: process 2 columns at a time
Index k = 0;
for (; k + 1 < actualPanelWidth; k += 2) {
@@ -107,13 +106,13 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
// Row i0: diagonal of col0 + contribution from col1
{
ResScalar r0 = s1 * cjl(c1[i0]);
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) r0 += s0 * cjl(c0[i0]);
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) r0 += s0 * cjl(c0[i0]);
res_[i0] += r0;
}
// Diagonal of column 1
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) res_[i1] += s1 * cjl(c1[i1]);
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) res_[i1] += s1 * cjl(c1[i1]);
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
res_[i0] += s0;
res_[i1] += s1;
}
@@ -123,8 +122,8 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
ResScalar s = alpha * cjr(rhs_[i * rhsIncr]);
const LhsScalar* EIGEN_RESTRICT c = lhs_ + i * lhsStride;
for (Index j = pi; j < i; ++j) res_[j] += s * cjl(c[j]);
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) res_[i] += s * cjl(c[i]);
EIGEN_IF_CONSTEXPR(HasUnitDiag) res_[i] += s;
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) res_[i] += s * cjl(c[i]);
EIGEN_IF_CONSTEXPR (HasUnitDiag) res_[i] += s;
}
}
@@ -137,7 +136,7 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
RhsMapper(&rhs_[pi * rhsIncr], rhsIncr), &res_[s], resIncr, alpha);
}
}
EIGEN_IF_CONSTEXPR(!IsLower) {
EIGEN_IF_CONSTEXPR (!IsLower) {
if (cols > size) {
general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjLhs, RhsScalar, RhsMapper, ConjRhs>::run(
rows, cols - size, LhsMapper(&lhs_[size * lhsStride], lhsStride), RhsMapper(&rhs_[size * rhsIncr], rhsIncr),
@@ -184,18 +183,17 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
const LhsScalar* EIGEN_RESTRICT row_i = lhs_ + i * lhsStride;
ResScalar dot = ResScalar(0);
EIGEN_IF_CONSTEXPR(IsLower) {
EIGEN_IF_CONSTEXPR (IsLower) {
Index s = pi;
Index len = (HasUnitDiag || HasZeroDiag) ? k : k + 1;
for (Index j = 0; j < len; ++j) dot += cjl(row_i[s + j]) * cjr(rhs_[s + j]);
}
else {
} else {
Index s = (HasUnitDiag || HasZeroDiag) ? i + 1 : i;
Index len = pi + actualPanelWidth - s;
for (Index j = 0; j < len; ++j) dot += cjl(row_i[s + j]) * cjr(rhs_[s + j]);
}
res_[i * resIncr] += alpha * dot;
EIGEN_IF_CONSTEXPR(HasUnitDiag) res_[i * resIncr] += alpha * cjr(rhs_[i]);
EIGEN_IF_CONSTEXPR (HasUnitDiag) res_[i * resIncr] += alpha * cjr(rhs_[i]);
}
// Rectangular part: delegate to optimized GEMV
@@ -207,7 +205,7 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
RhsMapper(&rhs_[s], rhsIncr), &res_[pi * resIncr], resIncr, alpha);
}
}
EIGEN_IF_CONSTEXPR(IsLower) {
EIGEN_IF_CONSTEXPR (IsLower) {
if (rows > diagSize) {
general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjLhs, RhsScalar, RhsMapper, ConjRhs>::run(
rows - diagSize, cols, LhsMapper(&lhs_[diagSize * lhsStride], lhsStride), RhsMapper(rhs_, rhsIncr),
@@ -98,7 +98,7 @@ EIGEN_STRONG_INLINE void trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageO
Index j = IsLower ? size - k - 1 : k;
typename LhsMapper::LinearMapper r = lhs.getLinearMapper(0, j);
EIGEN_IF_CONSTEXPR(OtherInnerStride == 1 && packet_traits<Scalar>::Vectorizable) {
EIGEN_IF_CONSTEXPR (OtherInnerStride == 1 && packet_traits<Scalar>::Vectorizable) {
using Packet = typename packet_traits<Scalar>::type;
constexpr Index PS = unpacket_traits<Packet>::size;
// Unrolled k3 loop by 4 to reduce r load/store traffic.
@@ -144,7 +144,7 @@ EIGEN_STRONG_INLINE void trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageO
for (; i < otherSize; ++i) r(i) -= a(i) * b;
}
// Vectorized diagonal scaling.
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == 0) {
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) {
Scalar inv_rjj = RealScalar(1) / conj(rhs(j, j));
Packet pinv = pset1<Packet>(inv_rjj);
Index i = 0;
@@ -153,14 +153,13 @@ EIGEN_STRONG_INLINE void trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageO
}
for (; i < otherSize; ++i) r(i) *= inv_rjj;
}
}
else {
} else {
for (Index k3 = 0; k3 < k; ++k3) {
Scalar b = conj(rhs(IsLower ? j + 1 + k3 : k3, j));
typename LhsMapper::LinearMapper a = lhs.getLinearMapper(0, IsLower ? j + 1 + k3 : k3);
for (Index i = 0; i < otherSize; ++i) r(i) -= a(i) * b;
}
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == 0) {
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) {
Scalar inv_rjj = RealScalar(1) / conj(rhs(j, j));
for (Index i = 0; i < otherSize; ++i) r(i) *= inv_rjj;
}
@@ -202,8 +201,8 @@ EIGEN_DONT_INLINE void triangular_solve_matrix<Scalar, Index, OnTheLeft, Mode, C
#if defined(EIGEN_VECTORIZE_AVX512) && defined(EIGEN_USE_AVX512_TRSM_L_KERNELS) && EIGEN_USE_AVX512_TRSM_L_KERNELS && \
EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS
EIGEN_IF_CONSTEXPR(
(OtherInnerStride == 1 && (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 &&
(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
// Very rough cutoffs to determine when to call trsm w/o packing
// For small problem sizes trsmKernel compiled with clang is generally faster.
// TODO: Investigate better heuristics for cutoffs.
@@ -270,8 +269,8 @@ EIGEN_DONT_INLINE void triangular_solve_matrix<Scalar, Index, OnTheLeft, Mode, C
{
Index i = IsLower ? k2 + k1 : k2 - k1;
#if defined(EIGEN_VECTORIZE_AVX512) && defined(EIGEN_USE_AVX512_TRSM_L_KERNELS) && EIGEN_USE_AVX512_TRSM_L_KERNELS
EIGEN_IF_CONSTEXPR(
(OtherInnerStride == 1 && (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 &&
(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
i = IsLower ? k2 + k1 : k2 - k1 - actualPanelWidth;
}
#endif
@@ -335,8 +334,8 @@ EIGEN_DONT_INLINE void triangular_solve_matrix<Scalar, Index, OnTheRight, Mode,
#if defined(EIGEN_VECTORIZE_AVX512) && defined(EIGEN_USE_AVX512_TRSM_R_KERNELS) && EIGEN_USE_AVX512_TRSM_R_KERNELS && \
EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS
EIGEN_IF_CONSTEXPR(
(OtherInnerStride == 1 && (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 &&
(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
// TODO: Investigate better heuristics for cutoffs.
std::ptrdiff_t l1, l2, l3;
manage_caching_sizes(GetAction, &l1, &l2, &l3);
+7 -7
View File
@@ -494,7 +494,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) {
if (size == 0) return nullptr; // short-cut. Also fixes Bug 884
check_size_for_overflow<T>(size);
T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * size));
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) {
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
EIGEN_TRY { default_construct_elements_of_array(result, size); }
EIGEN_CATCH(...) {
conditional_aligned_free<Align>(result);
@@ -506,7 +506,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) {
template <typename T, bool Align>
EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) {
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) {
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
return conditional_aligned_realloc_new<T, Align>(pts, new_size, old_size);
}
@@ -518,7 +518,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::si
template <typename T, bool Align>
EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T* ptr, std::size_t size) {
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) destruct_elements_of_array<T>(ptr, size);
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) destruct_elements_of_array<T>(ptr, size);
conditional_aligned_free<Align>(ptr);
}
@@ -673,12 +673,12 @@ class aligned_stack_memory_handler {
**/
EIGEN_DEVICE_FUNC aligned_stack_memory_handler(T* ptr, std::size_t size, bool dealloc)
: m_ptr(ptr), m_size(size), m_deallocate(dealloc) {
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) {
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
if (m_ptr) Eigen::internal::default_construct_elements_of_array(m_ptr, size);
}
}
EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() {
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) {
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
if (m_ptr) Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size);
}
if (m_deallocate) Eigen::internal::aligned_free(m_ptr);
@@ -718,14 +718,14 @@ struct local_nested_eval_wrapper<Xpr, NbEvaluations, true> {
: object(ptr == 0 ? reinterpret_cast<Scalar*>(Eigen::internal::aligned_malloc(sizeof(Scalar) * xpr.size())) : ptr,
xpr.rows(), xpr.cols()),
m_deallocate(ptr == 0) {
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::RequireInitialization) {
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
if (object.data()) Eigen::internal::default_construct_elements_of_array(object.data(), object.size());
}
object = xpr;
}
EIGEN_DEVICE_FUNC ~local_nested_eval_wrapper() {
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::RequireInitialization) {
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
if (object.data()) Eigen::internal::destruct_elements_of_array(object.data(), object.size());
}
if (m_deallocate) Eigen::internal::aligned_free(object.data());
+2 -3
View File
@@ -157,13 +157,12 @@ VectorwiseOp<ExpressionType, Direction>::cross(const MatrixBase<OtherDerived>& o
typename internal::nested_eval<OtherDerived, 2>::type vec(other.derived());
CrossReturnType res(_expression().rows(), _expression().cols());
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
eigen_assert(CrossReturnType::RowsAtCompileTime == 3 && "the matrix must have exactly 3 rows");
res.row(0) = (mat.row(1) * vec.coeff(2) - mat.row(2) * vec.coeff(1)).conjugate();
res.row(1) = (mat.row(2) * vec.coeff(0) - mat.row(0) * vec.coeff(2)).conjugate();
res.row(2) = (mat.row(0) * vec.coeff(1) - mat.row(1) * vec.coeff(0)).conjugate();
}
else {
} else {
eigen_assert(CrossReturnType::ColsAtCompileTime == 3 && "the matrix must have exactly 3 columns");
res.col(0) = (mat.col(1) * vec.coeff(2) - mat.col(2) * vec.coeff(1)).conjugate();
res.col(1) = (mat.col(2) * vec.coeff(0) - mat.col(0) * vec.coeff(2)).conjugate();
+1 -1
View File
@@ -1429,7 +1429,7 @@ struct transform_left_product_impl<Other, Mode, Options, Dim, HDim, Dim, Dim> {
typedef TransformType ResultType;
static EIGEN_DEVICE_FUNC ResultType run(const Other& other, const TransformType& tr) {
TransformType res;
EIGEN_IF_CONSTEXPR(Mode != int(AffineCompact)) res.matrix().row(Dim) = tr.matrix().row(Dim);
EIGEN_IF_CONSTEXPR (Mode != int(AffineCompact)) res.matrix().row(Dim) = tr.matrix().row(Dim);
res.matrix().template topRows<Dim>().noalias() = other * tr.matrix().template topRows<Dim>();
return res;
}
@@ -142,7 +142,7 @@ class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner<Scalar_>
LeastSquareDiagonalPreconditioner& factorize(const MatType& mat) {
// Compute the inverse squared-norm of each column of mat
m_invdiag.resize(mat.cols());
EIGEN_IF_CONSTEXPR(MatType::IsRowMajor) {
EIGEN_IF_CONSTEXPR (MatType::IsRowMajor) {
m_invdiag.setZero();
for (Index j = 0; j < mat.outerSize(); ++j) {
for (typename MatType::InnerIterator it(mat, j); it; ++it) m_invdiag(it.index()) += numext::abs2(it.value());
@@ -151,8 +151,7 @@ class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner<Scalar_>
RealScalar sum = numext::real(m_invdiag(j));
m_invdiag(j) = sum > RealScalar(0) ? RealScalar(1) / sum : RealScalar(1);
}
}
else {
} else {
for (Index j = 0; j < mat.outerSize(); ++j) {
RealScalar sum = mat.col(j).squaredNorm();
m_invdiag(j) = sum > RealScalar(0) ? RealScalar(1) / sum : RealScalar(1);
+22 -24
View File
@@ -785,31 +785,29 @@ JacobiSVD<MatrixType, Options>& JacobiSVD<MatrixType, Options>::compute_impl(con
// Non-blocking paths: apply rotations individually. The real and complex
// paths are kept separate to avoid any codegen impact from the complex
// preconditioner on GCC's optimization of the real inner loop.
else
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex) {
// Complex non-blocking sweep: condition each 2x2 block to be real before diagonalizing.
for (Index p = 1; p < n; ++p) {
for (Index q = 0; q < p; ++q) {
RealScalar threshold = numext::maxi<RealScalar>(considerAsZero, precision * maxDiagEntry);
if (abs(m_workMatrix.coeff(p, q)) > threshold || abs(m_workMatrix.coeff(q, p)) > threshold) {
finished = false;
if (internal::svd_precondition_2x2_block_to_be_real<MatrixType, Options>::run(m_workMatrix, *this, p, q,
maxDiagEntry)) {
JacobiRotation<RealScalar> j_left, j_right;
internal::real_2x2_jacobi_svd(m_workMatrix, p, q, &j_left, &j_right);
m_workMatrix.applyOnTheLeft(p, q, j_left);
if (computeU()) m_matrixU.applyOnTheRight(p, q, j_left.transpose());
m_workMatrix.applyOnTheRight(p, q, j_right);
if (computeV()) m_matrixV.applyOnTheRight(p, q, j_right);
maxDiagEntry = numext::maxi<RealScalar>(
maxDiagEntry,
numext::maxi<RealScalar>(abs(m_workMatrix.coeff(p, p)), abs(m_workMatrix.coeff(q, q))));
}
else EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
// Complex non-blocking sweep: condition each 2x2 block to be real before diagonalizing.
for (Index p = 1; p < n; ++p) {
for (Index q = 0; q < p; ++q) {
RealScalar threshold = numext::maxi<RealScalar>(considerAsZero, precision * maxDiagEntry);
if (abs(m_workMatrix.coeff(p, q)) > threshold || abs(m_workMatrix.coeff(q, p)) > threshold) {
finished = false;
if (internal::svd_precondition_2x2_block_to_be_real<MatrixType, Options>::run(m_workMatrix, *this, p, q,
maxDiagEntry)) {
JacobiRotation<RealScalar> j_left, j_right;
internal::real_2x2_jacobi_svd(m_workMatrix, p, q, &j_left, &j_right);
m_workMatrix.applyOnTheLeft(p, q, j_left);
if (computeU()) m_matrixU.applyOnTheRight(p, q, j_left.transpose());
m_workMatrix.applyOnTheRight(p, q, j_right);
if (computeV()) m_matrixV.applyOnTheRight(p, q, j_right);
maxDiagEntry = numext::maxi<RealScalar>(
maxDiagEntry,
numext::maxi<RealScalar>(abs(m_workMatrix.coeff(p, p)), abs(m_workMatrix.coeff(q, q))));
}
}
}
}
else {
} else {
// Real non-blocking sweep: diagonalize each 2x2 block directly.
RealScalar threshold = numext::maxi<RealScalar>(considerAsZero, precision * maxDiagEntry);
for (Index p = 1; p < n; ++p) {
@@ -840,7 +838,7 @@ JacobiSVD<MatrixType, Options>& JacobiSVD<MatrixType, Options>::compute_impl(con
// treated by svd_precondition_2x2_block_to_be_real, and the imaginary part
// of some diagonal entry might not be null.
bool diagonal_has_imaginary_part = false;
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex) {
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
diagonal_has_imaginary_part = abs(numext::imag(m_workMatrix.coeff(i, i))) > considerAsZero;
}
if (diagonal_has_imaginary_part) {
@@ -952,7 +950,7 @@ EIGEN_DONT_INLINE bool JacobiSVD<MatrixType, Options>::blocked_sweep(RealScalar
// Right operations (column scaling) are applied directly since column
// ops are contiguous in column-major layout.
bool doRealSvd = true;
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex) {
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
Scalar z;
// nn = ||(w_pp, w_qp)||_2, the norm of the first column of the 2x2 block.
RealScalar nn = sqrt(numext::abs2(blockBuffer.coeff(kBlockSize, kBlockSize)) +
@@ -1066,7 +1064,7 @@ EIGEN_DONT_INLINE bool JacobiSVD<MatrixType, Options>::blocked_sweep(RealScalar
notFinished = true;
bool doRealSvd = true;
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex) {
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
doRealSvd = internal::svd_precondition_2x2_block_to_be_real<MatrixType, Options>::run(m_workMatrix, *this, p,
q, maxDiagEntry);
}
+2 -2
View File
@@ -424,10 +424,10 @@ bool SVDBase<Derived>::allocate(Index rows, Index cols, unsigned int computation
m_diagSize.setValue(numext::mini(m_rows.value(), m_cols.value()));
m_singularValues.resize(m_diagSize.value());
EIGEN_IF_CONSTEXPR(RowsAtCompileTime == Dynamic) {
EIGEN_IF_CONSTEXPR (RowsAtCompileTime == Dynamic) {
m_matrixU.resize(m_rows.value(), m_computeFullU ? m_rows.value() : m_computeThinU ? m_diagSize.value() : 0);
}
EIGEN_IF_CONSTEXPR(ColsAtCompileTime == Dynamic) {
EIGEN_IF_CONSTEXPR (ColsAtCompileTime == Dynamic) {
m_matrixV.resize(m_cols.value(), m_computeFullV ? m_cols.value() : m_computeThinV ? m_diagSize.value() : 0);
}
@@ -867,7 +867,7 @@ void SimplicialCholeskyBase<Derived>::ordering(const MatrixType& a, ConstCholMat
const Index size = a.rows();
pmat = &ap;
// Note that ordering methods compute the inverse permutation
EIGEN_IF_CONSTEXPR((!std::is_same<OrderingType, NaturalOrdering<StorageIndex> >::value)) {
EIGEN_IF_CONSTEXPR ((!std::is_same<OrderingType, NaturalOrdering<StorageIndex> >::value)) {
{
CholMatrixType C;
constexpr bool kUseAMDFastPath = std::is_same<OrderingType, AMDOrdering<StorageIndex> >::value;
@@ -882,16 +882,15 @@ void SimplicialCholeskyBase<Derived>::ordering(const MatrixType& a, ConstCholMat
ap.resize(size, size);
internal::permute_symm_to_symm<UpLo, Upper, NonHermitian>(a, ap, m_P.indices().data());
}
else {
} else {
m_Pinv.resize(0);
m_P.resize(0);
EIGEN_IF_CONSTEXPR(int(UpLo) == int(Lower) || MatrixType::IsRowMajor) {
EIGEN_IF_CONSTEXPR (int(UpLo) == int(Lower) || MatrixType::IsRowMajor) {
// we have to transpose the lower part to to the upper one
ap.resize(size, size);
internal::permute_symm_to_symm<UpLo, Upper, NonHermitian>(a, ap, NULL);
}
else internal::simplicial_cholesky_grab_input<CholMatrixType, MatrixType>::run(a, pmat, ap);
} else
internal::simplicial_cholesky_grab_input<CholMatrixType, MatrixType>::run(a, pmat, ap);
}
}
@@ -376,8 +376,9 @@ void SimplicialCholeskyBase<Derived>::factorize_preordered(const CholMatrixType&
Index p = Lp[k] + nonZerosPerCol[k]++;
Li[p] = k; /* store L(k,k) = sqrt (d) in column k */
bool failed;
EIGEN_IF_CONSTEXPR(NonHermitian) { failed = d == RealScalar(0); }
else {
EIGEN_IF_CONSTEXPR (NonHermitian) {
failed = d == RealScalar(0);
} else {
failed = numext::real(d) <= RealScalar(0);
}
if (failed) {
+3 -3
View File
@@ -252,9 +252,9 @@ struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Sparse> {
template <typename DstXprType, typename SrcXprType, typename Functor, typename Weak>
struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Dense, Weak> {
static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) {
EIGEN_IF_CONSTEXPR(
(std::is_same<Functor, internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>>::value))
dst.setZero();
EIGEN_IF_CONSTEXPR ((std::is_same<Functor, internal::assign_op<typename DstXprType::Scalar,
typename SrcXprType::Scalar>>::value))
dst.setZero();
internal::evaluator<SrcXprType> srcEval(src);
resize_if_allowed(dst, src, func);
+2 -3
View File
@@ -204,11 +204,10 @@ class sparse_matrix_block_impl : public SparseCompressedBase<Block<SparseMatrixT
}
// update outer index pointers and innerNonZeros
EIGEN_IF_CONSTEXPR(IsVectorAtCompileTime) {
EIGEN_IF_CONSTEXPR (IsVectorAtCompileTime) {
if (!m_matrix.isCompressed()) matrix.innerNonZeroPtr()[m_outerStart] = StorageIndex(nnz);
matrix.outerIndexPtr()[m_outerStart] = StorageIndex(start);
}
else {
} else {
StorageIndex p = StorageIndex(start);
for (Index k = 0; k < m_outerSize.value(); ++k) {
StorageIndex nnz_k = internal::convert_index<StorageIndex>(tmp.innerVector(k).nonZeros());
+1 -1
View File
@@ -198,7 +198,7 @@ struct sparse_time_dense_product_impl<SparseLhsType, DenseRhsType, DenseResType,
const auto* innerNnz = mat.innerNonZeroPtr();
// The fast result pointer path requires contiguous ColMajor result layout.
// Transpose<ColMajor> reports innerStride()==1 but is actually RowMajor, so check both.
EIGEN_IF_CONSTEXPR(!(Res::Flags & RowMajorBit)) {
EIGEN_IF_CONSTEXPR (!(Res::Flags & RowMajorBit)) {
if (res.innerStride() == 1) {
for (Index c = 0; c < rhs.cols(); ++c) {
typename Res::Scalar* y = res.data() + c * res.outerStride();
+3 -2
View File
@@ -144,8 +144,9 @@ struct sparse_selfadjoint_diagonal_product_impl {
const StorageIndex outer = outerIndex(row, col);
const Index k = count[outer]++;
dest.innerIndexPtr()[k] = innerIndex(row, col);
EIGEN_IF_CONSTEXPR(ProductOrder == OnTheLeft) { dest.valuePtr()[k] = diagonal.coeff(row) * coeff; }
else {
EIGEN_IF_CONSTEXPR (ProductOrder == OnTheLeft) {
dest.valuePtr()[k] = diagonal.coeff(row) * coeff;
} else {
dest.valuePtr()[k] = coeff * diagonal.coeff(col);
}
}
+2 -3
View File
@@ -228,7 +228,7 @@ class SparseMatrixBase : public EigenBase<Derived> {
using Nested = typename Derived::Nested;
using NestedCleaned = internal::remove_all_t<Nested>;
EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) {
EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) {
Nested nm(m.derived());
internal::evaluator<NestedCleaned> thisEval(nm);
@@ -268,8 +268,7 @@ class SparseMatrixBase : public EigenBase<Derived> {
}
s << std::endl;
}
}
else {
} else {
Nested nm(m.derived());
internal::evaluator<NestedCleaned> thisEval(nm);
if (m.cols() == 1) {
+3 -5
View File
@@ -90,18 +90,16 @@ class SparseRefBase : public SparseMapBase<Derived> {
protected:
template <typename Expression>
void construct(Expression& expr) {
EIGEN_IF_CONSTEXPR(Expression::IsVectorAtCompileTime) {
EIGEN_IF_CONSTEXPR (Expression::IsVectorAtCompileTime) {
const Index offset = expr.outerIndexPtr() ? expr.outerIndexPtr()[0] : 0;
auto inner_index_ptr = expr.innerIndexPtr();
auto value_ptr = expr.valuePtr();
if (inner_index_ptr) inner_index_ptr += offset;
if (value_ptr) value_ptr += offset;
internal::construct_at<Base>(this, expr.size(), expr.nonZeros(), inner_index_ptr, value_ptr);
}
else if (expr.outerIndexPtr() == 0) {
} else if (expr.outerIndexPtr() == 0) {
internal::construct_at<Base>(this, expr.size(), expr.nonZeros(), expr.innerIndexPtr(), expr.valuePtr());
}
else {
} else {
internal::construct_at<Base>(this, expr.rows(), expr.cols(), expr.nonZeros(), expr.outerIndexPtr(),
expr.innerIndexPtr(), expr.valuePtr(), expr.innerNonZeroPtr());
}
+12 -11
View File
@@ -335,7 +335,9 @@ inline void sparse_selfadjoint_time_dense_product(const SparseLhsType& lhs, cons
typename DenseResType::Scalar res_j(0);
for (; (ProcessFirstHalf ? i && i.index() < j : i); ++i) {
LhsScalar lhs_ij = i.value();
EIGEN_IF_CONSTEXPR(!LhsIsRowMajor) { lhs_ij = numext::conj(lhs_ij); }
EIGEN_IF_CONSTEXPR (!LhsIsRowMajor) {
lhs_ij = numext::conj(lhs_ij);
}
res_j += lhs_ij * rhs.coeff(i.index(), k);
res(i.index(), k) += numext::conj(lhs_ij) * rhs_j;
}
@@ -456,9 +458,10 @@ void permute_symm_to_fullsymm(
Index r = it.row();
Index c = it.col();
Index ip = perm ? perm[i] : i;
EIGEN_IF_CONSTEXPR(Mode == int(Upper | Lower))
count[StorageOrderMatch ? jp : ip]++;
else if (r == c) count[ip]++;
EIGEN_IF_CONSTEXPR (Mode == int(Upper | Lower))
count[StorageOrderMatch ? jp : ip]++;
else if (r == c)
count[ip]++;
else if ((Mode == Lower && r > c) || (Mode == Upper && r < c)) {
count[ip]++;
count[jp]++;
@@ -483,18 +486,16 @@ void permute_symm_to_fullsymm(
StorageIndex jp = perm ? perm[j] : j;
StorageIndex ip = perm ? perm[i] : i;
EIGEN_IF_CONSTEXPR(Mode == int(Upper | Lower)) {
EIGEN_IF_CONSTEXPR (Mode == int(Upper | Lower)) {
Index k = count[StorageOrderMatch ? jp : ip]++;
dest.innerIndexPtr()[k] = StorageOrderMatch ? ip : jp;
dest.valuePtr()[k] = it.value();
}
else if (r == c) {
} else if (r == c) {
Index k = count[ip]++;
dest.innerIndexPtr()[k] = ip;
dest.valuePtr()[k] = it.value();
}
else if (((Mode & Lower) == Lower && r > c) || ((Mode & Upper) == Upper && r < c)) {
EIGEN_IF_CONSTEXPR(!StorageOrderMatch) std::swap(ip, jp);
} else if (((Mode & Lower) == Lower && r > c) || ((Mode & Upper) == Upper && r < c)) {
EIGEN_IF_CONSTEXPR (!StorageOrderMatch) std::swap(ip, jp);
Index k = count[jp]++;
dest.innerIndexPtr()[k] = ip;
dest.valuePtr()[k] = it.value();
@@ -556,7 +557,7 @@ void permute_symm_to_symm(const MatrixType& mat,
Index k = count[int(DstMode) == int(Lower) ? (std::min)(ip, jp) : (std::max)(ip, jp)]++;
dest.innerIndexPtr()[k] = int(DstMode) == int(Lower) ? (std::max)(ip, jp) : (std::min)(ip, jp);
EIGEN_IF_CONSTEXPR(!StorageOrderMatch) std::swap(ip, jp);
EIGEN_IF_CONSTEXPR (!StorageOrderMatch) std::swap(ip, jp);
if (((int(DstMode) == int(Lower) && ip < jp) || (int(DstMode) == int(Upper) && ip > jp)))
dest.valuePtr()[k] = (NonHermitian ? it.value() : numext::conj(it.value()));
else
@@ -38,8 +38,9 @@ static void sparse_sparse_product_with_pruning_impl(const Lhs& lhs, const Rhs& r
AmbiVector<ResScalar, StorageIndex> tempVector(rows);
// mimics a resizeByInnerOuter:
EIGEN_IF_CONSTEXPR(ResultType::IsRowMajor) { res.resize(cols, rows); }
else {
EIGEN_IF_CONSTEXPR (ResultType::IsRowMajor) {
res.resize(cols, rows);
} else {
res.resize(rows, cols);
}
+14 -13
View File
@@ -105,12 +105,11 @@ struct unary_evaluator<TriangularView<ArgType, Mode>, IteratorBased> : evaluator
: Base(xprEval.m_argImpl, outer),
m_returnOne(false),
m_containsDiag(Base::outer() < xprEval.m_arg.innerSize()) {
EIGEN_IF_CONSTEXPR(SkipFirst) {
EIGEN_IF_CONSTEXPR (SkipFirst) {
while ((*this) && ((HasUnitDiag || SkipDiag) ? this->index() <= outer : this->index() < outer))
Base::operator++();
EIGEN_IF_CONSTEXPR(HasUnitDiag) m_returnOne = m_containsDiag;
}
else EIGEN_IF_CONSTEXPR(HasUnitDiag) {
EIGEN_IF_CONSTEXPR (HasUnitDiag) m_returnOne = m_containsDiag;
} else EIGEN_IF_CONSTEXPR (HasUnitDiag) {
if ((!Base::operator bool()) || Base::index() >= Base::outer()) {
if (Base::operator bool()) Base::operator++();
m_returnOne = m_containsDiag;
@@ -119,14 +118,14 @@ struct unary_evaluator<TriangularView<ArgType, Mode>, IteratorBased> : evaluator
}
EIGEN_STRONG_INLINE InnerIterator& operator++() {
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
if (m_returnOne) {
m_returnOne = false;
return *this;
}
}
Base::operator++();
EIGEN_IF_CONSTEXPR(HasUnitDiag && !SkipFirst) {
EIGEN_IF_CONSTEXPR (HasUnitDiag && !SkipFirst) {
if ((!Base::operator bool()) || Base::index() >= Base::outer()) {
if (Base::operator bool()) Base::operator++();
m_returnOne = m_containsDiag;
@@ -136,13 +135,15 @@ struct unary_evaluator<TriangularView<ArgType, Mode>, IteratorBased> : evaluator
}
EIGEN_STRONG_INLINE operator bool() const {
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
if (m_returnOne) return true;
}
EIGEN_IF_CONSTEXPR(SkipFirst) { return Base::operator bool(); }
else {
EIGEN_IF_CONSTEXPR(SkipDiag) { return (Base::operator bool() && this->index() < this->outer()); }
else {
EIGEN_IF_CONSTEXPR (SkipFirst) {
return Base::operator bool();
} else {
EIGEN_IF_CONSTEXPR (SkipDiag) {
return (Base::operator bool() && this->index() < this->outer());
} else {
return (Base::operator bool() && this->index() <= this->outer());
}
}
@@ -151,13 +152,13 @@ struct unary_evaluator<TriangularView<ArgType, Mode>, IteratorBased> : evaluator
inline Index row() const { return (ArgType::Flags & RowMajorBit ? Base::outer() : this->index()); }
inline Index col() const { return (ArgType::Flags & RowMajorBit ? this->index() : Base::outer()); }
inline StorageIndex index() const {
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
if (m_returnOne) return internal::convert_index<StorageIndex>(Base::outer());
}
return Base::index();
}
inline Scalar value() const {
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
if (m_returnOne) return Scalar(1);
}
return Base::value();
+3 -2
View File
@@ -382,8 +382,9 @@ void materialize_selfadjoint_pattern(const SparsityPatternRef<StorageIndex>& A,
for (Index j = 0; j < n; ++j) {
const StorageIndex* a_col = A.inner + A.outer[j];
const Index a_nz = A.nonZeros(j);
EIGEN_IF_CONSTEXPR(IsLower) { a_split(j) = std::lower_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col; }
else {
EIGEN_IF_CONSTEXPR (IsLower) {
a_split(j) = std::lower_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col;
} else {
a_split(j) = std::upper_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col;
}
}
+17 -17
View File
@@ -44,8 +44,8 @@ struct sparse_solve_triangular_selector<Lhs, Rhs, Mode, Lower, RowMajor> {
if (lastIndex == i) break;
tmp = numext::madd<Scalar>(-lastVal, other.coeff(lastIndex, col), tmp);
}
EIGEN_IF_CONSTEXPR(Mode & UnitDiag)
other.coeffRef(i, col) = tmp;
EIGEN_IF_CONSTEXPR (Mode & UnitDiag)
other.coeffRef(i, col) = tmp;
else {
eigen_assert(lastIndex == i);
other.coeffRef(i, col) = tmp / lastVal;
@@ -69,19 +69,20 @@ struct sparse_solve_triangular_selector<Lhs, Rhs, Mode, Upper, RowMajor> {
Scalar l_ii(0);
LhsIterator it(lhsEval, i);
while (it && it.index() < i) ++it;
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) {
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) {
eigen_assert(it && it.index() == i);
l_ii = it.value();
++it;
}
else if (it && it.index() == i)++ it;
} else if (it && it.index() == i)
++it;
for (; it; ++it) {
tmp = numext::madd<Scalar>(-it.value(), other.coeff(it.index(), col), tmp);
}
EIGEN_IF_CONSTEXPR(Mode & UnitDiag)
other.coeffRef(i, col) = tmp;
else other.coeffRef(i, col) = tmp / l_ii;
EIGEN_IF_CONSTEXPR (Mode & UnitDiag)
other.coeffRef(i, col) = tmp;
else
other.coeffRef(i, col) = tmp / l_ii;
}
}
}
@@ -102,7 +103,7 @@ struct sparse_solve_triangular_selector<Lhs, Rhs, Mode, Lower, ColMajor> {
{
LhsIterator it(lhsEval, i);
while (it && it.index() < i) ++it;
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) {
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) {
eigen_assert(it && it.index() == i);
tmp /= it.value();
}
@@ -129,7 +130,7 @@ struct sparse_solve_triangular_selector<Lhs, Rhs, Mode, Upper, ColMajor> {
Scalar& tmp = other.coeffRef(i, col);
if (!numext::is_exactly_zero(tmp)) // optimization when other is actually sparse
{
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) {
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) {
// TODO: replace this with a binary search. make sure the binary search is safe for partially sorted
// elements
LhsIterator it(lhsEval, i);
@@ -210,21 +211,20 @@ struct sparse_solve_triangular_sparse_selector<Lhs, Rhs, Mode, UpLo, ColMajor> {
if (!numext::is_exactly_zero(ci)) {
// find
typename Lhs::InnerIterator it(lhs, i);
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) {
EIGEN_IF_CONSTEXPR(IsLower) {
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) {
EIGEN_IF_CONSTEXPR (IsLower) {
eigen_assert(it.index() == i);
ci /= it.value();
}
else ci /= lhs.coeff(i, i);
} else
ci /= lhs.coeff(i, i);
}
tempVector.restart();
EIGEN_IF_CONSTEXPR(IsLower) {
EIGEN_IF_CONSTEXPR (IsLower) {
if (it.index() == i) ++it;
for (; it; ++it) {
tempVector.coeffRef(it.index()) = numext::madd<Scalar>(-ci, it.value(), tempVector.coeffRef(it.index()));
}
}
else {
} else {
for (; it && it.index() < i; ++it) {
tempVector.coeffRef(it.index()) = numext::madd<Scalar>(-ci, it.value(), tempVector.coeffRef(it.index()));
}
+16 -15
View File
@@ -155,11 +155,14 @@ struct SluMatrix : SuperMatrix {
template <typename Scalar>
void setScalarType() {
EIGEN_IF_CONSTEXPR((std::is_same<Scalar, float>::value))
Dtype = SLU_S;
else EIGEN_IF_CONSTEXPR((std::is_same<Scalar, double>::value)) Dtype = SLU_D;
else EIGEN_IF_CONSTEXPR((std::is_same<Scalar, std::complex<float> >::value)) Dtype = SLU_C;
else EIGEN_IF_CONSTEXPR((std::is_same<Scalar, std::complex<double> >::value)) Dtype = SLU_Z;
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, float>::value))
Dtype = SLU_S;
else EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, double>::value))
Dtype = SLU_D;
else EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<float> >::value))
Dtype = SLU_C;
else EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<double> >::value))
Dtype = SLU_Z;
else {
eigen_assert(false && "Scalar type not supported by SuperLU");
}
@@ -187,12 +190,11 @@ struct SluMatrix : SuperMatrix {
static SluMatrix Map(SparseMatrixBase<MatrixType> &a_mat) {
MatrixType &mat(a_mat.derived());
SluMatrix res;
EIGEN_IF_CONSTEXPR((MatrixType::Flags & RowMajorBit) == RowMajorBit) {
EIGEN_IF_CONSTEXPR ((MatrixType::Flags & RowMajorBit) == RowMajorBit) {
res.setStorageType(SLU_NR);
res.nrow = internal::convert_index<int>(mat.cols());
res.ncol = internal::convert_index<int>(mat.rows());
}
else {
} else {
res.setStorageType(SLU_NC);
res.nrow = internal::convert_index<int>(mat.rows());
res.ncol = internal::convert_index<int>(mat.cols());
@@ -208,8 +210,8 @@ struct SluMatrix : SuperMatrix {
res.setScalarType<typename MatrixType::Scalar>();
// FIXME: the following type mapping is approximate.
EIGEN_IF_CONSTEXPR(int(MatrixType::Flags) & int(Upper)) res.Mtype = SLU_TRU;
EIGEN_IF_CONSTEXPR(int(MatrixType::Flags) & int(Lower)) res.Mtype = SLU_TRL;
EIGEN_IF_CONSTEXPR (int(MatrixType::Flags) & int(Upper)) res.Mtype = SLU_TRU;
EIGEN_IF_CONSTEXPR (int(MatrixType::Flags) & int(Lower)) res.Mtype = SLU_TRL;
eigen_assert(((int(MatrixType::Flags) & int(SelfAdjoint)) == 0) &&
"SelfAdjoint matrix shape not supported by SuperLU");
@@ -239,12 +241,11 @@ template <typename Derived>
struct SluMatrixMapHelper<SparseMatrixBase<Derived> > {
typedef Derived MatrixType;
static void run(MatrixType &mat, SluMatrix &res) {
EIGEN_IF_CONSTEXPR((MatrixType::Flags & RowMajorBit) == RowMajorBit) {
EIGEN_IF_CONSTEXPR ((MatrixType::Flags & RowMajorBit) == RowMajorBit) {
res.setStorageType(SLU_NR);
res.nrow = mat.cols();
res.ncol = mat.rows();
}
else {
} else {
res.setStorageType(SLU_NC);
res.nrow = mat.rows();
res.ncol = mat.cols();
@@ -260,8 +261,8 @@ struct SluMatrixMapHelper<SparseMatrixBase<Derived> > {
res.setScalarType<typename MatrixType::Scalar>();
// FIXME: the following type mapping is approximate.
EIGEN_IF_CONSTEXPR(MatrixType::Flags & Upper) res.Mtype = SLU_TRU;
EIGEN_IF_CONSTEXPR(MatrixType::Flags & Lower) res.Mtype = SLU_TRL;
EIGEN_IF_CONSTEXPR (MatrixType::Flags & Upper) res.Mtype = SLU_TRU;
EIGEN_IF_CONSTEXPR (MatrixType::Flags & Lower) res.Mtype = SLU_TRL;
eigen_assert(((MatrixType::Flags & SelfAdjoint) == 0) && "SelfAdjoint matrix shape not supported by SuperLU");
}
+3 -2
View File
@@ -197,8 +197,9 @@ class RunQueue {
std::atomic_thread_fence(std::memory_order_acquire);
continue;
}
EIGEN_IF_CONSTEXPR(NeedSizeEstimate) { return CalculateSize(front, back); }
else {
EIGEN_IF_CONSTEXPR (NeedSizeEstimate) {
return CalculateSize(front, back);
} else {
// This value will be 0 if the queue is empty, and undefined otherwise.
unsigned maybe_zero = ((front ^ back) & kMask2);
// Queue size estimate must agree with maybe zero check on the queue
+2 -2
View File
@@ -45,7 +45,7 @@ struct band_solve_triangular_selector<Index, Mode, LhsScalar, ConjLhs, RhsScalar
.cwiseProduct(other.col(col).segment(IsLower ? i - actual_k : i + 1, actual_k))
.sum();
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == 0) other.coeffRef(i, col) /= cjLhs(i, IsLower ? k : 0);
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) other.coeffRef(i, col) /= cjLhs(i, IsLower ? k : 0);
}
}
}
@@ -69,7 +69,7 @@ struct band_solve_triangular_selector<Index, Mode, LhsScalar, ConjLhs, RhsScalar
int actual_k = (std::min)(k, size - ii - 1);
int actual_start = IsLower ? 1 : k - actual_k;
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == 0) other.coeffRef(i, col) /= cjLhs(IsLower ? 0 : k, i);
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) other.coeffRef(i, col) /= cjLhs(IsLower ? 0 : k, i);
if (actual_k > 0)
other.col(col).segment(IsLower ? i + 1 : i - actual_k, actual_k) -=
+6 -2
View File
@@ -38,7 +38,9 @@ struct packed_triangular_matrix_vector_product<Index, Mode, LhsScalar, ConjLhs,
if (!(HasUnitDiag || HasZeroDiag) || (--r > 0)) {
ResMap(res + (IsLower ? s + i : 0), r) += alpha * cj(rhs[i]) * ConjLhsType(LhsMap(lhs + s, r));
}
EIGEN_IF_CONSTEXPR(HasUnitDiag) { res[i] += alpha * cj(rhs[i]); }
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
res[i] += alpha * cj(rhs[i]);
}
lhs += IsLower ? size - i : i + 1;
}
};
@@ -67,7 +69,9 @@ struct packed_triangular_matrix_vector_product<Index, Mode, LhsScalar, ConjLhs,
alpha *
(ConjLhsType(LhsMap(lhs + s, r)).cwiseProduct(ConjRhsType(RhsMap(rhs + (IsLower ? 0 : s + i), r)))).sum();
}
EIGEN_IF_CONSTEXPR(HasUnitDiag) { res[i] += alpha * cj(rhs[i]); }
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
res[i] += alpha * cj(rhs[i]);
}
lhs += IsLower ? i + 1 : size - i;
}
};
+2 -2
View File
@@ -34,7 +34,7 @@ struct packed_triangular_solve_vector<LhsScalar, RhsScalar, Index, OnTheLeft, Mo
rhs[i] -= (ConjLhsType(LhsMap(lhs + s, pi))
.cwiseProduct(Map<const Matrix<RhsScalar, Dynamic, 1> >(rhs + (IsLower ? 0 : i + 1), pi)))
.sum();
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? i : 0]);
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? i : 0]);
IsLower ? lhs += pi + 1 : lhs -= pi + 2;
}
}
@@ -53,7 +53,7 @@ struct packed_triangular_solve_vector<LhsScalar, RhsScalar, Index, OnTheLeft, Mo
for (Index pi = 0; pi < size; ++pi) {
Index i = IsLower ? pi : size - pi - 1;
Index r = size - pi - 1;
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? 0 : i]);
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? 0 : i]);
if (r > 0)
Map<Matrix<RhsScalar, Dynamic, 1> >(rhs + (IsLower ? i + 1 : 0), r) -=
rhs[i] * ConjLhsType(LhsMap(lhs + (IsLower ? 1 : 0), r));
@@ -92,12 +92,11 @@ struct unary_evaluator<CoherentPadOp<ArgType, SizeAtCompileTime>>
: m_arg(pad.nestedExpression()), m_argImpl(m_arg), m_size(pad.nestedExpression().size()) {}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
EIGEN_IF_CONSTEXPR(XprType::IsRowMajor) {
EIGEN_IF_CONSTEXPR (XprType::IsRowMajor) {
if (col < m_size.value()) {
return m_argImpl.coeff(1, col);
}
}
else {
} else {
if (row < m_size.value()) {
return m_argImpl.coeff(row, 1);
}
@@ -116,7 +115,9 @@ struct unary_evaluator<CoherentPadOp<ArgType, SizeAtCompileTime>>
EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
// AutoDiff scalar's derivative must be a vector, which is enforced by static assert.
// Defer to linear access for simplicity.
EIGEN_IF_CONSTEXPR(XprType::IsRowMajor) { return packet(col); }
EIGEN_IF_CONSTEXPR (XprType::IsRowMajor) {
return packet(col);
}
return packet(row);
}
@@ -94,8 +94,9 @@ template <typename Iterator, bool IsColMajor>
struct TripletComp {
typedef typename Iterator::value_type Triplet;
bool operator()(const Triplet& a, const Triplet& b) {
EIGEN_IF_CONSTEXPR(IsColMajor) { return ((a.col() == b.col() && a.row() < b.row()) || (a.col() < b.col())); }
else {
EIGEN_IF_CONSTEXPR (IsColMajor) {
return ((a.col() == b.col() && a.row() < b.row()) || (a.col() < b.col()));
} else {
return ((a.row() == b.row() && a.col() < b.col()) || (a.row() < b.row()));
}
}
+6 -8
View File
@@ -72,8 +72,8 @@ inline void GetDenseElt(const std::string& line, std::complex<RealScalar>& val)
template <typename Scalar>
inline void putMarketHeader(std::string& header, int sym) {
header = "%%MatrixMarket matrix coordinate ";
EIGEN_IF_CONSTEXPR(
(std::is_same<Scalar, std::complex<float> >::value || std::is_same<Scalar, std::complex<double> >::value)) {
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<float> >::value ||
std::is_same<Scalar, std::complex<double> >::value)) {
header += " complex";
if (sym == Symmetric)
header += " symmetric";
@@ -81,8 +81,7 @@ inline void putMarketHeader(std::string& header, int sym) {
header += " Hermitian";
else
header += " general";
}
else {
} else {
header += " real";
if (sym == Symmetric)
header += " symmetric";
@@ -340,11 +339,10 @@ bool saveMarketDense(const DenseType& mat, const std::string& filename) {
out.flags(std::ios_base::scientific);
out.precision(std::numeric_limits<RealScalar>::digits10 + 2);
EIGEN_IF_CONSTEXPR(
(std::is_same<Scalar, std::complex<float> >::value || std::is_same<Scalar, std::complex<double> >::value)) {
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<float> >::value ||
std::is_same<Scalar, std::complex<double> >::value)) {
out << "%%MatrixMarket matrix array complex general\n";
}
else {
} else {
out << "%%MatrixMarket matrix array real general\n";
}
out << mat.rows() << " " << mat.cols() << "\n";
@@ -179,12 +179,14 @@ class MatrixMarketIterator {
if (!getMarketHeader(curfile, m_sym, iscomplex, isvector)) continue;
if (isvector) continue;
if (!iscomplex) {
EIGEN_IF_CONSTEXPR(
(std::is_same<Scalar, std::complex<float> >::value || std::is_same<Scalar, std::complex<double> >::value))
continue;
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<float> >::value ||
std::is_same<Scalar, std::complex<double> >::value))
continue;
}
if (iscomplex) {
EIGEN_IF_CONSTEXPR((std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value)) { continue; }
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value)) {
continue;
}
}
// Get the matrix name
+3 -2
View File
@@ -363,8 +363,9 @@ class Tensor : public TensorBase<Tensor<Scalar_, NumIndices_, Options_, IndexTyp
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index linearizedIndex(const array<Index, NumIndices>& indices) const {
EIGEN_IF_CONSTEXPR(Options & RowMajor) { return m_storage.dimensions().IndexOfRowMajor(indices); }
else {
EIGEN_IF_CONSTEXPR (Options & RowMajor) {
return m_storage.dimensions().IndexOfRowMajor(indices);
} else {
return m_storage.dimensions().IndexOfColMajor(indices);
}
}
+4 -6
View File
@@ -214,11 +214,10 @@ struct TensorEvaluator<const TensorPairReducerOp<ReduceOp, Dims, ArgType>, Devic
m_impl(op.expression().index_pairs().reduce(op.reduce_dims(), op.reduce_op()), device),
m_return_dim(op.return_dim()) {
gen_strides(m_orig_impl.dimensions(), m_strides);
EIGEN_IF_CONSTEXPR(Layout == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (Layout == static_cast<int>(ColMajor)) {
const Index total_size = internal::array_prod(m_orig_impl.dimensions());
m_stride_mod = (m_return_dim < NumDims - 1) ? m_strides[m_return_dim + 1] : total_size;
}
else {
} else {
const Index total_size = internal::array_prod(m_orig_impl.dimensions());
m_stride_mod = (m_return_dim > 0) ? m_strides[m_return_dim - 1] : total_size;
}
@@ -257,13 +256,12 @@ struct TensorEvaluator<const TensorPairReducerOp<ReduceOp, Dims, ArgType>, Devic
// Calculate m_stride_div and m_stride_mod, which are used to
// calculate the value of an index w.r.t. the m_return_dim.
EIGEN_IF_CONSTEXPR(Layout == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (Layout == static_cast<int>(ColMajor)) {
strides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
strides[i] = strides[i - 1] * dims[i - 1];
}
}
else {
} else {
strides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
strides[i] = strides[i + 1] * dims[i + 1];
+15 -22
View File
@@ -34,13 +34,12 @@ EIGEN_ALWAYS_INLINE DSizes<IndexType, NumDims> strides(const DSizes<IndexType, N
// TODO(ezhulenev): Use templates to unroll this loop (similar to
// h_array_reduce in MoreMeta.h)? Benchmark it.
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
strides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
strides[i] = strides[i - 1] * dimensions[i - 1];
}
}
else {
} else {
strides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
strides[i] = strides[i + 1] * dimensions[i + 1];
@@ -952,7 +951,7 @@ class StridedLinearBufferCopy {
const Scalar* src = &src_data[src_offset];
Scalar* dst = &dst_data[dst_offset];
EIGEN_IF_CONSTEXPR(!Vectorizable) {
EIGEN_IF_CONSTEXPR (!Vectorizable) {
for (Index i = 0; i < count; ++i) {
dst[i * dst_stride] = src[i * src_stride];
}
@@ -962,7 +961,7 @@ class StridedLinearBufferCopy {
const IndexType vectorized_size = PacketSize * (count / PacketSize);
IndexType i = 0;
EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Linear) {
EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Linear) {
// ******************************************************************** //
// Linear copy from `src` to `dst`.
const IndexType unrolled_size = (4 * PacketSize) * (count / (4 * PacketSize));
@@ -977,7 +976,7 @@ class StridedLinearBufferCopy {
Packet p = ploadu<Packet>(src + i);
pstoreu<Scalar, Packet>(dst + i, p);
}
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
if (i < vectorized_half_size) {
HalfPacket p = ploadu<HalfPacket>(src + i);
@@ -989,15 +988,14 @@ class StridedLinearBufferCopy {
dst[i] = src[i];
}
// ******************************************************************** //
}
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Scatter) {
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Scatter) {
// Scatter from `src` to `dst`.
eigen_assert(src_stride == 1 && dst_stride != 1);
for (; i < vectorized_size; i += PacketSize) {
Packet p = ploadu<Packet>(src + i);
pscatter<Scalar, Packet>(dst + i * dst_stride, p, dst_stride);
}
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
if (i < vectorized_half_size) {
HalfPacket p = ploadu<HalfPacket>(src + i);
@@ -1009,8 +1007,7 @@ class StridedLinearBufferCopy {
dst[i * dst_stride] = src[i];
}
// ******************************************************************** //
}
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::FillLinear) {
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::FillLinear) {
// Fill `dst` with value at `*src`.
eigen_assert(src_stride == 0 && dst_stride == 1);
@@ -1025,7 +1022,7 @@ class StridedLinearBufferCopy {
for (; i < vectorized_size; i += PacketSize) {
pstoreu<Scalar, Packet>(dst + i, p);
}
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
if (i < vectorized_half_size) {
HalfPacket hp = pset1<HalfPacket>(s);
@@ -1037,8 +1034,7 @@ class StridedLinearBufferCopy {
dst[i] = s;
}
// ******************************************************************** //
}
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::FillScatter) {
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::FillScatter) {
// Scatter `*src` into `dst`.
eigen_assert(src_stride == 0 && dst_stride != 1);
Scalar s = *src;
@@ -1046,7 +1042,7 @@ class StridedLinearBufferCopy {
for (; i < vectorized_size; i += PacketSize) {
pscatter<Scalar, Packet>(dst + i * dst_stride, p, dst_stride);
}
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
if (i < vectorized_half_size) {
HalfPacket hp = pset1<HalfPacket>(s);
@@ -1058,15 +1054,14 @@ class StridedLinearBufferCopy {
dst[i * dst_stride] = s;
}
// ******************************************************************** //
}
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Gather) {
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Gather) {
// Gather from `src` into `dst`.
eigen_assert(dst_stride == 1);
for (; i < vectorized_size; i += PacketSize) {
Packet p = pgather<Scalar, Packet>(src + i * src_stride, src_stride);
pstoreu<Scalar, Packet>(dst + i, p);
}
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
if (i < vectorized_half_size) {
HalfPacket p = pgather<Scalar, HalfPacket>(src + i * src_stride, src_stride);
@@ -1078,14 +1073,12 @@ class StridedLinearBufferCopy {
dst[i] = src[i * src_stride];
}
// ******************************************************************** //
}
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Random) {
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Random) {
// Random.
for (; i < count; ++i) {
dst[i * dst_stride] = src[i * src_stride];
}
}
else {
} else {
eigen_assert(false);
}
}
@@ -145,15 +145,14 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
}
}
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_inputStrides[0] = 1;
m_outputStrides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1];
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
}
}
else {
} else {
m_inputStrides[NumDims - 1] = 1;
m_outputStrides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
@@ -218,14 +217,13 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
return m_impl.coeff(0);
}
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
if (isCopy) {
return m_impl.coeff(index);
} else {
return coeffColMajor(index);
}
}
else {
} else {
if (isCopy) {
return m_impl.coeff(index);
} else {
@@ -314,7 +312,7 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
return internal::pset1<PacketReturnType>(m_impl.coeff(0));
}
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
if (isCopy) {
#ifdef EIGEN_GPU_COMPILE_PHASE
// See PR 437: on NVIDIA P100 and K20m we observed a x3-4 speed up by enforcing
@@ -332,8 +330,7 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
} else {
return packetColMajor<LoadMode>(index);
}
}
else {
} else {
if (isCopy) {
#ifdef EIGEN_GPU_COMPILE_PHASE
// See above.
@@ -361,11 +358,10 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
Index startDim, endDim;
Index inputIndex, outputOffset, batchedIndex;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
startDim = NumDims - 1;
endDim = 1;
}
else {
} else {
startDim = 0;
endDim = NumDims - 2;
}
@@ -156,13 +156,12 @@ struct TensorEvaluator<const TensorChippingOp<DimId, ArgType>, Device> {
m_stride = 1;
m_inputStride = 1;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = 0; i < m_dim.actualDim(); ++i) {
m_stride *= input_dims[i];
m_inputStride *= input_dims[i];
}
}
else {
} else {
for (int i = NumInputDims - 1; i > m_dim.actualDim(); --i) {
m_stride *= input_dims[i];
m_inputStride *= input_dims[i];
@@ -183,11 +182,10 @@ struct TensorEvaluator<const TensorChippingOp<DimId, ArgType>, Device> {
before_chipped_dim_product *= input_dims[i];
}
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_isEffectivelyInnerChipping = before_chipped_dim_product == 1;
m_isEffectivelyOuterChipping = after_chipped_dim_product == 1;
}
else {
} else {
m_isEffectivelyInnerChipping = after_chipped_dim_product == 1;
m_isEffectivelyOuterChipping = before_chipped_dim_product == 1;
}
@@ -164,7 +164,7 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
}
}
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_leftStrides[0] = 1;
m_rightStrides[0] = 1;
m_outputStrides[0] = 1;
@@ -174,8 +174,7 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
m_rightStrides[j] = m_rightStrides[j - 1] * rhs_dims[j - 1];
m_outputStrides[j] = m_outputStrides[j - 1] * m_dimensions[j - 1];
}
}
else {
} else {
m_leftStrides[NumDims - 1] = 1;
m_rightStrides[NumDims - 1] = 1;
m_outputStrides[NumDims - 1] = 1;
@@ -361,14 +360,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
// Collect dimension-wise indices (subs).
array<Index, NumDims> subs;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > 0; --i) {
subs[i] = index / m_outputStrides[i];
index -= subs[i] * m_outputStrides[i];
}
subs[0] = index;
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
subs[i] = index / m_outputStrides[i];
index -= subs[i] * m_outputStrides[i];
@@ -379,14 +377,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
const Dimensions& left_dims = m_leftImpl.dimensions();
if (subs[m_axis] < left_dims[m_axis]) {
Index left_index;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
left_index = subs[0];
EIGEN_UNROLL_LOOP
for (int i = 1; i < NumDims; ++i) {
left_index += (subs[i] % left_dims[i]) * m_leftStrides[i];
}
}
else {
} else {
left_index = subs[NumDims - 1];
EIGEN_UNROLL_LOOP
for (int i = NumDims - 2; i >= 0; --i) {
@@ -398,14 +395,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
subs[m_axis] -= left_dims[m_axis];
const Dimensions& right_dims = m_rightImpl.dimensions();
Index right_index;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
right_index = subs[0];
EIGEN_UNROLL_LOOP
for (int i = 1; i < NumDims; ++i) {
right_index += (subs[i] % right_dims[i]) * m_rightStrides[i];
}
}
else {
} else {
right_index = subs[NumDims - 1];
EIGEN_UNROLL_LOOP
for (int i = NumDims - 2; i >= 0; --i) {
@@ -436,7 +432,7 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
array<Index, NumDims> subs_end;
Index remaining = index;
Index remaining_end = index + packetSize - 1;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > 0; --i) {
subs[i] = remaining / m_outputStrides[i];
remaining -= subs[i] * m_outputStrides[i];
@@ -445,8 +441,7 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
}
subs[0] = remaining;
subs_end[0] = remaining_end;
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
subs[i] = remaining / m_outputStrides[i];
remaining -= subs[i] * m_outputStrides[i];
@@ -476,14 +471,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
if (on_left) {
Index left_index;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
left_index = subs[0];
EIGEN_UNROLL_LOOP
for (int i = 1; i < NumDims; ++i) {
left_index += subs[i] * m_leftStrides[i];
}
}
else {
} else {
left_index = subs[NumDims - 1];
EIGEN_UNROLL_LOOP
for (int i = NumDims - 2; i >= 0; --i) {
@@ -495,14 +489,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
if (on_right) {
subs[m_axis] -= left_axis_size;
Index right_index;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
right_index = subs[0];
EIGEN_UNROLL_LOOP
for (int i = 1; i < NumDims; ++i) {
right_index += subs[i] * m_rightStrides[i];
}
}
else {
} else {
right_index = subs[NumDims - 1];
EIGEN_UNROLL_LOOP
for (int i = NumDims - 2; i >= 0; --i) {
@@ -466,7 +466,7 @@ struct TensorContractionEvaluatorBase {
DSizes<Index, LDims> eval_left_dims;
DSizes<Index, RDims> eval_right_dims;
array<IndexPair<Index>, ContractDims> eval_op_indices;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
// For ColMajor, we keep using the existing dimensions
for (int i = 0; i < LDims; i++) {
eval_left_dims[i] = m_leftImpl.dimensions()[i];
@@ -479,8 +479,7 @@ struct TensorContractionEvaluatorBase {
eval_op_indices[i].first = op.indices()[i].first;
eval_op_indices[i].second = op.indices()[i].second;
}
}
else {
} else {
// For RowMajor, we need to reverse the existing dimensions
for (int i = 0; i < LDims; i++) {
eval_left_dims[i] = m_leftImpl.dimensions()[LDims - i - 1];
@@ -642,7 +641,7 @@ struct TensorContractionEvaluatorBase {
}
// If the layout is RowMajor, we need to reverse the m_dimensions
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(RowMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(RowMajor)) {
for (int i = 0, j = NumDims - 1; i < j; i++, j--) {
numext::swap(m_dimensions[i], m_dimensions[j]);
}
@@ -265,13 +265,12 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT
internal::tensor_contraction_dispatch(
[&](auto lhs_c, auto rhs_c, auto rhs_r) {
EIGEN_IF_CONSTEXPR(IsEvalInSyncMode) {
EIGEN_IF_CONSTEXPR (IsEvalInSyncMode) {
EvalParallelContext<NoCallback, lhs_c(), rhs_c(), rhs_r(), Alignment> ctx(
this, num_threads, buffer, m, n, k, bm, bn, bk, nm, nn, nk, gm, gn, nm0, nn0, shard_by_col,
parallel_pack, parallelize_by_sharding_dim_only, NoCallback());
ctx.run();
}
else {
} else {
auto* ctx = new EvalParallelContext<DoneCallback, lhs_c(), rhs_c(), rhs_r(), Alignment>(
this, num_threads, buffer, m, n, k, bm, bn, bk, nm, nn, nk, gm, gn, nm0, nn0, shard_by_col,
parallel_pack, parallelize_by_sharding_dim_only, std::move(done));
@@ -34,15 +34,14 @@ class IndexMapper {
array<Index, NumDims> inputStrides;
array<Index, NumDims> outputStrides;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
inputStrides[0] = 1;
outputStrides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
inputStrides[i] = inputStrides[i - 1] * input_dims[i - 1];
outputStrides[i] = outputStrides[i - 1] * dimensions[i - 1];
}
}
else {
} else {
inputStrides[NumDims - 1] = 1;
outputStrides[NumDims - 1] = 1;
for (int i = static_cast<int>(NumDims) - 2; i >= 0; --i) {
@@ -79,7 +78,7 @@ class IndexMapper {
m_outputStrides[i] = outputStrides[ordering[i]];
}
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = 0; i < NumDims; ++i) {
if (i > NumKernelDims) {
m_gpuInputStrides[i] = m_gpuInputStrides[i - 1] * gpuInputDimensions[i - 1];
@@ -89,8 +88,7 @@ class IndexMapper {
m_gpuOutputStrides[i] = 1;
}
}
}
else {
} else {
for (int i = NumDims - 1; i >= 0; --i) {
if (i + 1 < static_cast<int>(offset)) {
m_gpuInputStrides[i] = m_gpuInputStrides[i + 1] * gpuInputDimensions[i + 1];
@@ -105,7 +103,7 @@ class IndexMapper {
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputPlaneToTensorInputOffset(Index p) const {
Index inputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int d = NumDims - 1; d > NumKernelDims; --d) {
const Index idx = p / m_gpuInputStrides[d];
inputIndex += idx * m_inputStrides[d];
@@ -114,8 +112,7 @@ class IndexMapper {
if (NumKernelDims < NumDims) {
inputIndex += p * m_inputStrides[NumKernelDims];
}
}
else {
} else {
std::ptrdiff_t limit = 0;
if (NumKernelDims < NumDims) {
limit = NumDims - NumKernelDims - 1;
@@ -132,7 +129,7 @@ class IndexMapper {
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputPlaneToTensorOutputOffset(Index p) const {
Index outputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int d = NumDims - 1; d > NumKernelDims; --d) {
const Index idx = p / m_gpuOutputStrides[d];
outputIndex += idx * m_outputStrides[d];
@@ -141,8 +138,7 @@ class IndexMapper {
if (NumKernelDims < NumDims) {
outputIndex += p * m_outputStrides[NumKernelDims];
}
}
else {
} else {
std::ptrdiff_t limit = 0;
if (NumKernelDims < NumDims) {
limit = NumDims - NumKernelDims - 1;
@@ -314,13 +310,12 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
const typename TensorEvaluator<InputArgType, Device>::Dimensions& input_dims = m_inputImpl.dimensions();
const typename TensorEvaluator<KernelArgType, Device>::Dimensions& kernel_dims = m_kernelImpl.dimensions();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_inputStride[0] = 1;
for (int i = 1; i < NumDims; ++i) {
m_inputStride[i] = m_inputStride[i - 1] * input_dims[i - 1];
}
}
else {
} else {
m_inputStride[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
m_inputStride[i] = m_inputStride[i + 1] * input_dims[i + 1];
@@ -328,7 +323,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
}
m_dimensions = m_inputImpl.dimensions();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = 0; i < NumKernelDims; ++i) {
const Index index = op.indices()[i];
const Index input_dim = input_dims[index];
@@ -347,8 +342,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
for (int i = 1; i < NumDims; ++i) {
m_outputStride[i] = m_outputStride[i - 1] * m_dimensions[i - 1];
}
}
else {
} else {
for (int i = NumKernelDims - 1; i >= 0; --i) {
const Index index = op.indices()[i];
const Index input_dim = input_dims[index];
@@ -404,7 +398,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
EIGEN_DEVICE_FUNC PacketReturnType packet(const Index index) const {
Index indices[2] = {index, index + PacketSize - 1};
Index startInputs[2] = {0, 0};
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > 0; --i) {
const Index idx0 = indices[0] / m_outputStride[i];
const Index idx1 = indices[1] / m_outputStride[i];
@@ -413,8 +407,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
indices[0] -= idx0 * m_outputStride[i];
indices[1] -= idx1 * m_outputStride[i];
}
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx0 = indices[0] / m_outputStride[i];
const Index idx1 = indices[1] / m_outputStride[i];
@@ -462,14 +455,13 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
private:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const {
Index startInput = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = index / m_outputStride[i];
startInput += idx * m_inputStride[i];
index -= idx * m_outputStride[i];
}
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / m_outputStride[i];
startInput += idx * m_inputStride[i];
@@ -120,20 +120,18 @@ struct TensorEvaluator {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<DenseIndex, NumCoords>& coords) const {
eigen_assert(m_data != NULL);
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
return m_data[m_dims.IndexOfColMajor(coords)];
}
else {
} else {
return m_data[m_dims.IndexOfRowMajor(coords)];
}
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(const array<DenseIndex, NumCoords>& coords) const {
eigen_assert(m_data != NULL);
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
return m_data[m_dims.IndexOfColMajor(coords)];
}
else {
} else {
return m_data[m_dims.IndexOfRowMajor(coords)];
}
}
+4 -6
View File
@@ -179,13 +179,12 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D
m_dimensions[i] = input_dims[i];
}
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_strides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1];
}
}
else {
} else {
m_strides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1];
@@ -644,7 +643,7 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index getBaseOffsetFromIndex(Index index, Index omitted_dim) const {
Index result = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > omitted_dim; --i) {
const Index partial_m_stride = m_strides[i] / m_dimensions[omitted_dim];
const Index idx = index / partial_m_stride;
@@ -652,8 +651,7 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D
result += idx * m_strides[i];
}
result += index;
}
else {
} else {
for (Index i = 0; i < omitted_dim; ++i) {
const Index partial_m_stride = m_strides[i] / m_dimensions[omitted_dim];
const Index idx = index / partial_m_stride;
@@ -203,8 +203,9 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_,
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index linearizedIndex(const array<Index, NumIndices>& indices) const {
EIGEN_IF_CONSTEXPR(Options & RowMajor) { return m_storage.dimensions().IndexOfRowMajor(indices); }
else {
EIGEN_IF_CONSTEXPR (Options & RowMajor) {
return m_storage.dimensions().IndexOfRowMajor(indices);
} else {
return m_storage.dimensions().IndexOfColMajor(indices);
}
}
@@ -105,15 +105,14 @@ struct TensorEvaluator<const TensorGeneratorOp<Generator, ArgType>, Device> {
TensorEvaluator<ArgType, Device> argImpl(op.expression(), device);
m_dimensions = argImpl.dimensions();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_strides[0] = 1;
EIGEN_UNROLL_LOOP
for (int i = 1; i < NumDims; ++i) {
m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1];
if (m_strides[i] != 0) m_fast_strides[i] = IndexDivisor(m_strides[i]);
}
}
else {
} else {
m_strides[NumDims - 1] = 1;
EIGEN_UNROLL_LOOP
for (int i = NumDims - 2; i >= 0; --i) {
@@ -242,15 +241,14 @@ struct TensorEvaluator<const TensorGeneratorOp<Generator, ArgType>, Device> {
protected:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void extract_coordinates(Index index, array<Index, NumDims>& coords) const {
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = index / m_fast_strides[i];
index -= idx * m_strides[i];
coords[i] = idx;
}
coords[0] = index;
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / m_fast_strides[i];
index -= idx * m_strides[i];
+6 -4
View File
@@ -238,8 +238,9 @@ struct TensorPrinter {
if (fmt.precision == StreamPrecision) {
explicit_precision = 0;
} else if (fmt.precision == FullPrecision) {
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsInteger) { explicit_precision = 0; }
else {
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsInteger) {
explicit_precision = 0;
} else {
explicit_precision = significant_decimals_impl<Scalar>::run();
}
} else {
@@ -385,8 +386,9 @@ struct TensorPrinter<Tensor, 0, Format> {
if (fmt.precision == StreamPrecision) {
explicit_precision = 0;
} else if (fmt.precision == FullPrecision) {
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsInteger) { explicit_precision = 0; }
else {
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsInteger) {
explicit_precision = 0;
} else {
explicit_precision = significant_decimals_impl<Scalar>::run();
}
} else {
@@ -193,12 +193,11 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
// Caches a few variables.
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_inputDepth = input_dims[0];
m_inputRows = input_dims[1];
m_inputCols = input_dims[2];
}
else {
} else {
m_inputDepth = input_dims[NumInputDims - 1];
m_inputRows = input_dims[NumInputDims - 2];
m_inputCols = input_dims[NumInputDims - 3];
@@ -270,7 +269,7 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
eigen_assert(m_outputCols > 0);
// Dimensions for result of extraction.
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
// ColMajor
// 0: depth
// 1: patch_rows
@@ -284,8 +283,7 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
for (int i = 4; i < NumDims; ++i) {
m_dimensions[i] = input_dims[i - 1];
}
}
else {
} else {
// RowMajor
// NumDims-1: depth
// NumDims-2: patch_rows
@@ -302,12 +300,11 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
}
// Strides for moving the patch in various dimensions.
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_colStride = m_dimensions[1];
m_patchStride = m_colStride * m_dimensions[2] * m_dimensions[0];
m_otherStride = m_patchStride * m_dimensions[3];
}
else {
} else {
m_colStride = m_dimensions[NumDims - 2];
m_patchStride = m_colStride * m_dimensions[NumDims - 3] * m_dimensions[NumDims - 1];
m_otherStride = m_patchStride * m_dimensions[NumDims - 4];
@@ -328,10 +325,9 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
// Number of patches in the width dimension.
m_fastOutputRows = internal::TensorIntDivisor<Index>(m_outputRows);
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[0]);
}
else {
} else {
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[NumDims - 1]);
}
}
@@ -111,15 +111,14 @@ struct TensorEvaluator<const TensorInflationOp<Strides, ArgType>, Device> {
}
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_outputStrides[0] = 1;
m_inputStrides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1];
}
}
else { // RowMajor
} else { // RowMajor
m_outputStrides[NumDims - 1] = 1;
m_inputStrides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
@@ -142,7 +141,7 @@ struct TensorEvaluator<const TensorInflationOp<Strides, ArgType>, Device> {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool getInputIndex(Index index, Index* inputIndex) const {
eigen_assert(index < dimensions().TotalSize());
*inputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = index / m_outputStrides[i];
@@ -157,8 +156,7 @@ struct TensorEvaluator<const TensorInflationOp<Strides, ArgType>, Device> {
}
*inputIndex += index / m_strides[0];
return true;
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / m_outputStrides[i];
+8 -12
View File
@@ -105,11 +105,10 @@ class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_, MakePoi
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StoragePointerType data() const { return m_data; }
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageRefType operator()(const array<Index, NumIndices>& indices) const {
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
const Index index = m_dimensions.IndexOfRowMajor(indices);
return m_data[index];
}
else {
} else {
const Index index = m_dimensions.IndexOfColMajor(indices);
return m_data[index];
}
@@ -130,12 +129,11 @@ class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_, MakePoi
IndexTypes... otherIndices) const {
EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 2 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE)
eigen_assert(internal::all((Eigen::NumTraits<Index>::highest() >= otherIndices)...));
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
const Index index =
m_dimensions.IndexOfRowMajor(array<Index, NumIndices>{{firstIndex, secondIndex, otherIndices...}});
return m_data[index];
}
else {
} else {
const Index index =
m_dimensions.IndexOfColMajor(array<Index, NumIndices>{{firstIndex, secondIndex, otherIndices...}});
return m_data[index];
@@ -143,11 +141,10 @@ class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_, MakePoi
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageRefType operator()(const array<Index, NumIndices>& indices) {
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
const Index index = m_dimensions.IndexOfRowMajor(indices);
return m_data[index];
}
else {
} else {
const Index index = m_dimensions.IndexOfColMajor(indices);
return m_data[index];
}
@@ -170,12 +167,11 @@ class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_, MakePoi
"Number of indices used to access a tensor coefficient must be equal to the rank of the tensor.");
eigen_assert(internal::all((Eigen::NumTraits<Index>::highest() >= otherIndices)...));
const std::size_t NumDims = sizeof...(otherIndices) + 2;
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
const Index index =
m_dimensions.IndexOfRowMajor(array<Index, NumDims>{{firstIndex, secondIndex, otherIndices...}});
return m_data[index];
}
else {
} else {
const Index index =
m_dimensions.IndexOfColMajor(array<Index, NumDims>{{firstIndex, secondIndex, otherIndices...}});
return m_data[index];
+16 -24
View File
@@ -399,7 +399,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
const Sizes& output_dims = op.sizes();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_inputStrides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1];
@@ -411,8 +411,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
m_outputStrides[i] = m_outputStrides[i - 1] * output_dims[i - 1];
m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1);
}
}
else {
} else {
m_inputStrides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
m_inputStrides[i] = m_inputStrides[i + 1] * input_dims[i + 1];
@@ -433,15 +432,14 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
m_impl.evalSubExprsIfNeeded(NULL);
if (!NumTraits<std::remove_const_t<Scalar>>::RequireInitialization && data && m_impl.data()) {
Index contiguous_values = 1;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = 0; i < NumDims; ++i) {
contiguous_values *= dimensions()[i];
if (dimensions()[i] != m_impl.dimensions()[i]) {
break;
}
}
}
else {
} else {
for (int i = NumDims - 1; i >= 0; --i) {
contiguous_values *= dimensions()[i];
if (dimensions()[i] != m_impl.dimensions()[i]) {
@@ -493,7 +491,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
Index inputIndices[] = {0, 0};
Index indices[] = {index, index + packetSize - 1};
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
const Index idx0 = indices[0] / m_fastOutputStrides[i];
@@ -505,8 +503,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
}
inputIndices[0] += (indices[0] + m_offsets[0]);
inputIndices[1] += (indices[1] + m_offsets[0]);
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx0 = indices[0] / m_fastOutputStrides[i];
@@ -557,7 +554,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
typename Storage::Type result = constCast(m_impl.data());
if (result) {
Index offset = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = 0; i < NumDims; ++i) {
if (m_dimensions[i] != m_impl.dimensions()[i]) {
offset += m_offsets[i] * m_inputStrides[i];
@@ -570,8 +567,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
break;
}
}
}
else {
} else {
for (int i = NumDims - 1; i >= 0; --i) {
if (m_dimensions[i] != m_impl.dimensions()[i]) {
offset += m_offsets[i] * m_inputStrides[i];
@@ -593,7 +589,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
protected:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
Index inputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = index / m_fastOutputStrides[i];
@@ -601,8 +597,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
index -= idx * m_outputStrides[i];
}
inputIndex += (index + m_offsets[0]);
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / m_fastOutputStrides[i];
@@ -675,7 +670,7 @@ struct TensorEvaluator<TensorSlicingOp<StartIndices, Sizes, ArgType>, Device>
const int packetSize = PacketType<CoeffReturnType, Device>::size;
Index inputIndices[] = {0, 0};
Index indices[] = {index, index + packetSize - 1};
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
const Index idx0 = indices[0] / this->m_fastOutputStrides[i];
@@ -687,8 +682,7 @@ struct TensorEvaluator<TensorSlicingOp<StartIndices, Sizes, ArgType>, Device>
}
inputIndices[0] += (indices[0] + this->m_offsets[0]);
inputIndices[1] += (indices[1] + this->m_offsets[0]);
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx0 = indices[0] / this->m_fastOutputStrides[i];
@@ -844,7 +838,7 @@ struct TensorEvaluator<const TensorStridingSlicingOp<StartIndices, StopIndices,
Strides output_dims = m_dimensions;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_inputStrides[0] = m_strides[0];
m_offsets[0] = startIndicesClamped[0];
Index previousDimProduct = 1;
@@ -860,8 +854,7 @@ struct TensorEvaluator<const TensorStridingSlicingOp<StartIndices, StopIndices,
m_outputStrides[i] = m_outputStrides[i - 1] * output_dims[i - 1];
m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1);
}
}
else {
} else {
m_inputStrides[NumDims - 1] = m_strides[NumDims - 1];
m_offsets[NumDims - 1] = startIndicesClamped[NumDims - 1];
Index previousDimProduct = 1;
@@ -905,15 +898,14 @@ struct TensorEvaluator<const TensorStridingSlicingOp<StartIndices, StopIndices,
protected:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
Index inputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i >= 0; --i) {
const Index idx = index / m_fastOutputStrides[i];
inputIndex += idx * m_inputStrides[i] + m_offsets[i];
index -= idx * m_outputStrides[i];
}
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims; ++i) {
const Index idx = index / m_fastOutputStrides[i];
+9 -10
View File
@@ -121,7 +121,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
m_dimensions[i] += m_padding[i].first + m_padding[i].second;
}
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_inputStrides[0] = 1;
m_outputStrides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
@@ -129,8 +129,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
}
m_outputStrides[NumDims] = m_outputStrides[NumDims - 1] * m_dimensions[NumDims - 1];
}
else {
} else {
m_inputStrides[NumDims - 1] = 1;
m_outputStrides[NumDims] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
@@ -160,7 +159,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
eigen_assert(index < dimensions().TotalSize());
Index inputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = index / m_outputStrides[i];
@@ -174,8 +173,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
return m_paddingValue;
}
inputIndex += (index - m_padding[0].first);
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / m_outputStrides[i + 1];
@@ -195,17 +193,18 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
template <int LoadMode>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) { return packetColMajor(index); }
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
return packetColMajor(index);
}
return packetRowMajor(index);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
TensorOpCost cost = m_impl.costPerCoeff(vectorized);
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims; ++i) updateCostPerDimension(cost, i, i == 0);
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i >= 0; --i) updateCostPerDimension(cost, i, i == NumDims - 1);
}
+6 -9
View File
@@ -101,7 +101,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
Index num_patches = 1;
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
const PatchDim& patch_dims = op.patch_dims();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = 0; i < NumDims - 1; ++i) {
m_dimensions[i] = patch_dims[i];
num_patches *= (input_dims[i] - patch_dims[i] + 1);
@@ -118,8 +118,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
for (int i = 1; i < NumDims; ++i) {
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
}
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
m_dimensions[i + 1] = patch_dims[i];
num_patches *= (input_dims[i] - patch_dims[i] + 1);
@@ -155,7 +154,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
// Find the offset of the element wrt the location of the first element.
Index patchOffset = index - patchIndex * m_outputStrides[output_stride_index];
Index inputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 2; i > 0; --i) {
const Index patchIdx = patchIndex / m_patchStrides[i];
@@ -164,8 +163,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
patchOffset -= offsetIdx * m_outputStrides[i];
inputIndex += (patchIdx + offsetIdx) * m_inputStrides[i];
}
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 2; ++i) {
const Index patchIdx = patchIndex / m_patchStrides[i];
@@ -191,7 +189,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
indices[1] - patchIndices[1] * m_outputStrides[output_stride_index]};
Index inputIndices[2] = {0, 0};
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 2; i > 0; --i) {
const Index patchIdx[2] = {patchIndices[0] / m_patchStrides[i], patchIndices[1] / m_patchStrides[i]};
@@ -205,8 +203,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
inputIndices[0] += (patchIdx[0] + offsetIdx[0]) * m_inputStrides[i];
inputIndices[1] += (patchIdx[1] + offsetIdx[1]) * m_inputStrides[i];
}
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 2; ++i) {
const Index patchIdx[2] = {patchIndices[0] / m_patchStrides[i], patchIndices[1] / m_patchStrides[i]};
+12 -18
View File
@@ -607,14 +607,13 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
// Precompute output strides.
if (NumOutputDims > 0) {
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_outputStrides[0] = 1;
for (int i = 1; i < NumOutputDims; ++i) {
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i]);
}
}
else {
} else {
m_outputStrides[static_cast<size_t>(NumOutputDims - 1)] = 1;
for (int i = NumOutputDims - 2; i >= 0; --i) {
m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1];
@@ -626,13 +625,12 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
// Precompute input strides.
if (NumInputDims > 0) {
array<Index, NumInputDims> input_strides;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
input_strides[0] = 1;
for (int i = 1; i < NumInputDims; ++i) {
input_strides[i] = input_strides[i - 1] * input_dims[i - 1];
}
}
else {
} else {
input_strides.back() = 1;
for (int i = NumInputDims - 2; i >= 0; --i) {
input_strides[i] = input_strides[i + 1] * input_dims[i + 1];
@@ -695,10 +693,9 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
else if ((RunningOnGPU && (m_device.majorDeviceVersion() >= 3)) || (RunningOnSycl)) {
bool reducing_inner_dims = true;
for (int i = 0; i < NumReducedDims; ++i) {
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
reducing_inner_dims &= m_reduced[i];
}
else {
} else {
reducing_inner_dims &= m_reduced[NumInputDims - 1 - i];
}
}
@@ -733,10 +730,9 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
bool preserving_inner_dims = true;
for (int i = 0; i < NumReducedDims; ++i) {
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
preserving_inner_dims &= m_reduced[NumInputDims - 1 - i];
}
else {
} else {
preserving_inner_dims &= m_reduced[i];
}
}
@@ -946,16 +942,15 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
// used to compute the reduction at output index "index".
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const {
if (ReducingInnerMostDims) {
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
return index * m_preservedStrides[0];
}
else {
} else {
return index * m_preservedStrides[NumPreservedStrides - 1];
}
}
// TBD: optimize the case where we preserve the innermost dimensions.
Index startInput = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumOutputDims - 1; i > 0; --i) {
// This is index_i in the output tensor.
const Index idx = index / m_outputStrides[i];
@@ -968,8 +963,7 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
} else {
startInput += index * m_preservedStrides[0];
}
}
else {
} else {
for (int i = 0; i < NumOutputDims - 1; ++i) {
// This is index_i in the output tensor.
const Index idx = index / m_outputStrides[i];
@@ -26,7 +26,7 @@ namespace internal {
// updated the content of the output address it will try again.
template <typename T, typename R>
__device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) {
EIGEN_IF_CONSTEXPR(sizeof(T) == 4) {
EIGEN_IF_CONSTEXPR (sizeof(T) == 4) {
unsigned int oldval = *reinterpret_cast<unsigned int*>(output);
unsigned int newval = oldval;
reducer.reduce(accum, reinterpret_cast<T*>(&newval));
@@ -42,8 +42,7 @@ __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer)
return;
}
}
}
else EIGEN_IF_CONSTEXPR(sizeof(T) == 8) {
} else EIGEN_IF_CONSTEXPR (sizeof(T) == 8) {
unsigned long long oldval = *reinterpret_cast<unsigned long long*>(output);
unsigned long long newval = oldval;
reducer.reduce(accum, reinterpret_cast<T*>(&newval));
@@ -59,8 +58,7 @@ __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer)
return;
}
}
}
else {
} else {
gpu_assert(0 && "Wordsize not supported");
}
}
@@ -367,8 +365,9 @@ template <typename Op>
__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void ReductionCleanupKernelHalfFloat(Op reducer, half* output, half* scratch) {
eigen_assert(threadIdx.x == 1);
typedef packet_traits<Eigen::half>::type packet_type;
EIGEN_IF_CONSTEXPR(unpacket_traits<packet_type>::size == 1) { *output = *scratch; }
else {
EIGEN_IF_CONSTEXPR (unpacket_traits<packet_type>::size == 1) {
*output = *scratch;
} else {
half2* pscratch = reinterpret_cast<half2*>(scratch);
half tmp = __float2half(0.f);
for (int i = 0; i < unpacket_traits<packet_type>::size; i += 2) {
+4 -6
View File
@@ -194,13 +194,12 @@ class TensorRefBase : public TensorBase<Derived> {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(const array<Index, NumIndices>& indices) const {
const Dimensions& dims = this->dimensions();
Index index = 0;
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
index += indices[0];
for (size_t i = 1; i < NumIndices; ++i) {
index = index * dims[i] + indices[i];
}
}
else {
} else {
index += indices[NumIndices - 1];
for (int i = NumIndices - 2; i >= 0; --i) {
index = index * dims[i] + indices[i];
@@ -278,13 +277,12 @@ class TensorRef : public internal::TensorRefBase<TensorRef<PlainObjectType>> {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(const array<Index, NumIndices>& indices) {
const Dimensions& dims = this->dimensions();
Index index = 0;
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
index += indices[0];
for (size_t i = 1; i < NumIndices; ++i) {
index = index * dims[i] + indices[i];
}
}
else {
} else {
index += indices[NumIndices - 1];
for (int i = NumIndices - 2; i >= 0; --i) {
index = index * dims[i] + indices[i];
+4 -6
View File
@@ -116,14 +116,13 @@ struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device
// Compute strides
m_dimensions = m_impl.dimensions();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_strides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1];
if (m_strides[i] > 0) m_fastStrides[i] = IndexDivisor(m_strides[i]);
}
}
else {
} else {
m_strides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1];
@@ -151,7 +150,7 @@ struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index reverseIndex(Index index) const {
eigen_assert(index < dimensions().TotalSize());
Index inputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
Index idx = index / m_fastStrides[i];
@@ -166,8 +165,7 @@ struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device
} else {
inputIndex += index;
}
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
Index idx = index / m_fastStrides[i];
+6 -9
View File
@@ -111,14 +111,13 @@ struct TensorEvaluator<const TensorRollOp<RollDimensions, ArgType>, Device> {
// Compute strides
m_dimensions = m_impl.dimensions();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_strides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1];
if (m_strides[i] > 0) m_fast_strides[i] = IndexDivisor(m_strides[i]);
}
}
else {
} else {
m_strides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1];
@@ -164,7 +163,7 @@ struct TensorEvaluator<const TensorRollOp<RollDimensions, ArgType>, Device> {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rollIndex(Index index) const {
eigen_assert(index < dimensions().TotalSize());
Index rolledIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
Index idx = index / m_fast_strides[i];
@@ -172,8 +171,7 @@ struct TensorEvaluator<const TensorRollOp<RollDimensions, ArgType>, Device> {
rolledIndex += roll(idx, m_rolls[i], m_dimensions[i]) * m_strides[i];
}
rolledIndex += roll(index, m_rolls[0], m_dimensions[0]);
}
else {
} else {
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
Index idx = index / m_fast_strides[i];
@@ -307,15 +305,14 @@ struct TensorEvaluator<const TensorRollOp<RollDimensions, ArgType>, Device> {
const Device EIGEN_DEVICE_REF m_device;
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void extract_coordinates(Index index, array<Index, NumDims>& coords) const {
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = index / m_fast_strides[i];
index -= idx * m_strides[i];
coords[i] = idx;
}
coords[0] = index;
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / m_fast_strides[i];
index -= idx * m_strides[i];
+2 -3
View File
@@ -398,12 +398,11 @@ struct TensorEvaluator<const TensorScanOp<Op, ArgType>, Device> {
// Compute stride of scan axis
const Dimensions& dims = m_impl.dimensions();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = 0; i < op.axis(); ++i) {
m_stride = m_stride * dims[i];
}
}
else {
} else {
// dims can only be indexed through unsigned integers,
// so use an unsigned type to let the compiler know.
// This prevents spurious warnings: "'*((void*)(& evaluator)+64)[18446744073709551615]' may be used uninitialized
@@ -120,7 +120,7 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
}
}
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_unshuffledInputStrides[0] = 1;
m_outputStrides[0] = 1;
@@ -130,8 +130,7 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
m_fastOutputStrides[i] =
internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : Index(1));
}
}
else {
} else {
m_unshuffledInputStrides[NumDims - 1] = 1;
m_outputStrides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
@@ -265,15 +264,14 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
const DSizes<Index, NumDims>& output_block_strides,
const DSizes<internal::TensorIntDivisor<Index>, NumDims>& fast_input_block_strides) const {
Index output_index = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = input_index / fast_input_block_strides[i];
output_index += idx * output_block_strides[m_inverseShuffle[i]];
input_index -= idx * input_block_strides[i];
}
return output_index + input_index * output_block_strides[m_inverseShuffle[0]];
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = input_index / fast_input_block_strides[i];
output_index += idx * output_block_strides[m_inverseShuffle[i]];
@@ -285,15 +283,14 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
Index inputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = index / m_fastOutputStrides[i];
inputIndex += idx * m_inputStrides[i];
index -= idx * m_outputStrides[i];
}
return inputIndex + index * m_inputStrides[0];
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / m_fastOutputStrides[i];
inputIndex += idx * m_inputStrides[i];
+8 -12
View File
@@ -107,7 +107,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
}
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_outputStrides[0] = 1;
m_inputStrides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
@@ -116,8 +116,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
m_inputStrides[i - 1] *= op.strides()[i - 1];
}
m_inputStrides[NumDims - 1] *= op.strides()[NumDims - 1];
}
else { // RowMajor
} else { // RowMajor
m_outputStrides[NumDims - 1] = 1;
m_inputStrides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
@@ -148,7 +147,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
Index inputIndices[] = {0, 0};
Index indices[] = {index, index + PacketSize - 1};
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
const Index idx0 = indices[0] / m_outputStrides[i];
@@ -160,8 +159,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
}
inputIndices[0] += indices[0] * m_inputStrides[0];
inputIndices[1] += indices[1] * m_inputStrides[0];
}
else { // RowMajor
} else { // RowMajor
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx0 = indices[0] / m_outputStrides[i];
@@ -208,7 +206,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
protected:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
Index inputIndex = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = index / m_outputStrides[i];
@@ -216,8 +214,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
index -= idx * m_outputStrides[i];
}
inputIndex += index * m_inputStrides[0];
}
else { // RowMajor
} else { // RowMajor
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / m_outputStrides[i];
@@ -271,7 +268,7 @@ struct TensorEvaluator<TensorStridingOp<Strides, ArgType>, Device>
Index inputIndices[] = {0, 0};
Index indices[] = {index, index + PacketSize - 1};
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_UNROLL_LOOP
for (int i = NumDims - 1; i > 0; --i) {
const Index idx0 = indices[0] / this->m_outputStrides[i];
@@ -283,8 +280,7 @@ struct TensorEvaluator<TensorStridingOp<Strides, ArgType>, Device>
}
inputIndices[0] += indices[0] * this->m_inputStrides[0];
inputIndices[1] += indices[1] * this->m_inputStrides[0];
}
else { // RowMajor
} else { // RowMajor
EIGEN_UNROLL_LOOP
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx0 = indices[0] / this->m_outputStrides[i];
+6 -9
View File
@@ -159,13 +159,12 @@ struct TensorEvaluator<const TensorTraceOp<Dims, ArgType>, Device> {
// Compute the output strides
if (NumOutputDims > 0) {
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_outputStrides[0] = 1;
for (int i = 1; i < NumOutputDims; ++i) {
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
}
}
else {
} else {
m_outputStrides.back() = 1;
for (int i = NumOutputDims - 2; i >= 0; --i) {
m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1];
@@ -176,13 +175,12 @@ struct TensorEvaluator<const TensorTraceOp<Dims, ArgType>, Device> {
// Compute the input strides
if (NumInputDims > 0) {
array<Index, NumInputDims> input_strides;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
input_strides[0] = 1;
for (int i = 1; i < NumInputDims; ++i) {
input_strides[i] = input_strides[i - 1] * input_dims[i - 1];
}
}
else {
} else {
input_strides.back() = 1;
for (int i = NumInputDims - 2; i >= 0; --i) {
input_strides[i] = input_strides[i + 1] * input_dims[i + 1];
@@ -249,15 +247,14 @@ struct TensorEvaluator<const TensorTraceOp<Dims, ArgType>, Device> {
// Given the output index, finds the first index in the input tensor used to compute the trace
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const {
Index startInput = 0;
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumOutputDims - 1; i > 0; --i) {
const Index idx = index / m_outputStrides[i];
startInput += idx * m_preservedStrides[i];
index -= idx * m_outputStrides[i];
}
startInput += index * m_preservedStrides[0];
}
else {
} else {
for (int i = 0; i < NumOutputDims - 1; ++i) {
const Index idx = index / m_outputStrides[i];
startInput += idx * m_preservedStrides[i];
@@ -209,13 +209,12 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
// Cache a few variables.
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_inputDepth = input_dims[0];
m_inputPlanes = input_dims[1];
m_inputRows = input_dims[2];
m_inputCols = input_dims[3];
}
else {
} else {
m_inputDepth = input_dims[NumInputDims - 1];
m_inputPlanes = input_dims[NumInputDims - 2];
m_inputRows = input_dims[NumInputDims - 3];
@@ -288,7 +287,7 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
eigen_assert(m_outputPlanes > 0);
// Dimensions for result of extraction.
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
// ColMajor
// 0: depth
// 1: patch_planes
@@ -304,8 +303,7 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
for (int i = 5; i < NumDims; ++i) {
m_dimensions[i] = input_dims[i - 1];
}
}
else {
} else {
// RowMajor
// NumDims-1: depth
// NumDims-2: patch_planes
@@ -324,13 +322,12 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
}
// Strides for the output tensor.
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_rowStride = m_dimensions[1];
m_colStride = m_dimensions[2] * m_rowStride;
m_patchStride = m_colStride * m_dimensions[3] * m_dimensions[0];
m_otherStride = m_patchStride * m_dimensions[4];
}
else {
} else {
m_rowStride = m_dimensions[NumDims - 2];
m_colStride = m_dimensions[NumDims - 3] * m_rowStride;
m_patchStride = m_colStride * m_dimensions[NumDims - 4] * m_dimensions[NumDims - 1];
@@ -358,10 +355,9 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
m_fastOutputPlanes = internal::TensorIntDivisor<Index>(m_outputPlanes);
m_fastOutputPlanesRows = internal::TensorIntDivisor<Index>(m_outputPlanesRows);
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[0]);
}
else {
} else {
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[NumDims - 1]);
}
}
+2 -2
View File
@@ -75,8 +75,8 @@ void sparse_extra(const SparseMatrixType& ref) {
// test coeff and coeffRef
for (int i = 0; i < (int)zeroCoords.size(); ++i) {
VERIFY_IS_MUCH_SMALLER_THAN(m.coeff(zeroCoords[i].x(), zeroCoords[i].y()), eps);
EIGEN_IF_CONSTEXPR((std::is_same<SparseMatrixType, SparseMatrix<Scalar, Flags> >::value))
VERIFY_RAISES_ASSERT(m.coeffRef(zeroCoords[0].x(), zeroCoords[0].y()) = 5);
EIGEN_IF_CONSTEXPR ((std::is_same<SparseMatrixType, SparseMatrix<Scalar, Flags> >::value))
VERIFY_RAISES_ASSERT(m.coeffRef(zeroCoords[0].x(), zeroCoords[0].y()) = 5);
}
VERIFY_IS_APPROX(m, refMat);
+22 -33
View File
@@ -144,15 +144,14 @@ template <int Layout, int NumDims>
static Index GetInputIndex(Index output_index, const array<Index, NumDims>& output_to_input_dim_map,
const array<Index, NumDims>& input_strides, const array<Index, NumDims>& output_strides) {
int input_index = 0;
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = output_index / output_strides[i];
input_index += idx * input_strides[output_to_input_dim_map[i]];
output_index -= idx * output_strides[i];
}
return input_index + output_index * input_strides[output_to_input_dim_map[0]];
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = output_index / output_strides[i];
input_index += idx * input_strides[output_to_input_dim_map[i]];
@@ -165,13 +164,12 @@ static Index GetInputIndex(Index output_index, const array<Index, NumDims>& outp
template <int Layout, int NumDims>
static array<Index, NumDims> ComputeStrides(const array<Index, NumDims>& sizes) {
array<Index, NumDims> strides;
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
strides[0] = 1;
for (int i = 1; i < NumDims; ++i) {
strides[i] = strides[i - 1] * sizes[i - 1];
}
}
else {
} else {
strides[NumDims - 1] = 1;
for (int i = NumDims - 2; i >= 0; --i) {
strides[i] = strides[i + 1] * sizes[i + 1];
@@ -228,7 +226,7 @@ static void test_uniform_block_shape() {
// Test shape 'UniformAllDims' with larger 'max_coeff count' which spills
// partially into first inner-most dimension.
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 7 * 5 * 5 * 5 * 5;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
@@ -238,8 +236,7 @@ static void test_uniform_block_shape() {
VERIFY_IS_EQUAL(5, block.dimensions()[i]);
}
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
}
else {
} else {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 5 * 5 * 5 * 5 * 6;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
@@ -253,7 +250,7 @@ static void test_uniform_block_shape() {
// Test shape 'UniformAllDims' with larger 'max_coeff count' which spills
// fully into first inner-most dimension.
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 11 * 5 * 5 * 5 * 5;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
@@ -263,8 +260,7 @@ static void test_uniform_block_shape() {
VERIFY_IS_EQUAL(5, block.dimensions()[i]);
}
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
}
else {
} else {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 5 * 5 * 5 * 5 * 7;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
@@ -278,7 +274,7 @@ static void test_uniform_block_shape() {
// Test shape 'UniformAllDims' with larger 'max_coeff count' which spills
// fully into first few inner-most dimensions.
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
DSizes<Index, 5> dims(7, 5, 6, 17, 7);
const Index max_coeff_count = 7 * 5 * 6 * 7 * 5;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
@@ -289,8 +285,7 @@ static void test_uniform_block_shape() {
VERIFY_IS_EQUAL(7, block.dimensions()[3]);
VERIFY_IS_EQUAL(5, block.dimensions()[4]);
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
}
else {
} else {
DSizes<Index, 5> dims(7, 5, 6, 9, 7);
const Index max_coeff_count = 5 * 5 * 5 * 6 * 7;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
@@ -304,7 +299,7 @@ static void test_uniform_block_shape() {
}
// Test shape 'UniformAllDims' with full allocation to all dims.
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
DSizes<Index, 5> dims(7, 5, 6, 17, 7);
const Index max_coeff_count = 7 * 5 * 6 * 17 * 7;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
@@ -315,8 +310,7 @@ static void test_uniform_block_shape() {
VERIFY_IS_EQUAL(17, block.dimensions()[3]);
VERIFY_IS_EQUAL(7, block.dimensions()[4]);
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
}
else {
} else {
DSizes<Index, 5> dims(7, 5, 6, 9, 7);
const Index max_coeff_count = 7 * 5 * 6 * 9 * 7;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
@@ -336,7 +330,7 @@ static void test_skewed_inner_dim_block_shape() {
typedef internal::TensorBlockMapper<5, Layout> TensorBlockMapper;
// Test shape 'SkewedInnerDims' with partial allocation to inner-most dim.
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 10 * 1 * 1 * 1 * 1;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
@@ -346,8 +340,7 @@ static void test_skewed_inner_dim_block_shape() {
VERIFY_IS_EQUAL(1, block.dimensions()[i]);
}
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
}
else {
} else {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 1 * 1 * 1 * 1 * 6;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
@@ -360,7 +353,7 @@ static void test_skewed_inner_dim_block_shape() {
}
// Test shape 'SkewedInnerDims' with full allocation to inner-most dim.
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 11 * 1 * 1 * 1 * 1;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
@@ -370,8 +363,7 @@ static void test_skewed_inner_dim_block_shape() {
VERIFY_IS_EQUAL(1, block.dimensions()[i]);
}
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
}
else {
} else {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 1 * 1 * 1 * 1 * 7;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
@@ -385,7 +377,7 @@ static void test_skewed_inner_dim_block_shape() {
// Test shape 'SkewedInnerDims' with full allocation to inner-most dim,
// and partial allocation to second inner-dim.
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 11 * 3 * 1 * 1 * 1;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
@@ -396,8 +388,7 @@ static void test_skewed_inner_dim_block_shape() {
VERIFY_IS_EQUAL(1, block.dimensions()[i]);
}
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
}
else {
} else {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 1 * 1 * 1 * 15 * 7;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
@@ -412,7 +403,7 @@ static void test_skewed_inner_dim_block_shape() {
// Test shape 'SkewedInnerDims' with full allocation to inner-most dim,
// and partial allocation to third inner-dim.
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 11 * 5 * 5 * 1 * 1;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
@@ -424,8 +415,7 @@ static void test_skewed_inner_dim_block_shape() {
VERIFY_IS_EQUAL(1, block.dimensions()[i]);
}
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
}
else {
} else {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 1 * 1 * 5 * 17 * 7;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
@@ -440,7 +430,7 @@ static void test_skewed_inner_dim_block_shape() {
}
// Test shape 'SkewedInnerDims' with full allocation to all dims.
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 11 * 5 * 6 * 17 * 7;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
@@ -451,8 +441,7 @@ static void test_skewed_inner_dim_block_shape() {
VERIFY_IS_EQUAL(17, block.dimensions()[3]);
VERIFY_IS_EQUAL(7, block.dimensions()[4]);
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
}
else {
} else {
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
const Index max_coeff_count = 11 * 5 * 6 * 17 * 7;
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
+4 -6
View File
@@ -17,10 +17,9 @@ using Eigen::internal::TensorExecutor;
template <typename T, int NumDims, int Layout>
static void setRandomForBinaryProduct(Tensor<T, NumDims, Layout>& tensor) {
EIGEN_IF_CONSTEXPR((std::is_integral<T>::value && !std::is_same<T, bool>::value)) {
EIGEN_IF_CONSTEXPR ((std::is_integral<T>::value && !std::is_same<T, bool>::value)) {
setRandomDataInRange(tensor, T(-1000), T(1000));
}
else {
} else {
tensor.setRandom();
}
}
@@ -86,15 +85,14 @@ static TensorBlockParams<NumDims> SkewedInnerBlock(DSizes<Index, NumDims> dims)
// Compute offsets for the first block coefficient.
Index index = block.offset();
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = index / strides[i];
index -= idx * strides[i];
offsets[i] = idx;
}
if (NumDims > 0) offsets[0] = index;
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = index / strides[i];
index -= idx * strides[i];
+2 -3
View File
@@ -38,15 +38,14 @@ template <int Layout, int NumDims>
static Index GetInputIndex(Index output_index, const array<Index, NumDims>& output_to_input_dim_map,
const array<Index, NumDims>& input_strides, const array<Index, NumDims>& output_strides) {
int input_index = 0;
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
for (int i = NumDims - 1; i > 0; --i) {
const Index idx = output_index / output_strides[i];
input_index += idx * input_strides[output_to_input_dim_map[i]];
output_index -= idx * output_strides[i];
}
return input_index + output_index * input_strides[output_to_input_dim_map[0]];
}
else {
} else {
for (int i = 0; i < NumDims - 1; ++i) {
const Index idx = output_index / output_strides[i];
input_index += idx * input_strides[output_to_input_dim_map[i]];