clang-format: register EIGEN_IF_CONSTEXPR as an IfMacro
libeigen/eigen!2604 Co-authored-by: Rasmus Munk Larsen <rmlarsen@gmail.com>
This commit is contained in:
co-authored by
Rasmus Munk Larsen
parent
7b56b05698
commit
7966ea495e
@@ -9,6 +9,8 @@ StatementMacros:
|
||||
- EIGEN_STATIC_ASSERT
|
||||
- EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED
|
||||
- EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN
|
||||
IfMacros:
|
||||
- EIGEN_IF_CONSTEXPR
|
||||
SortIncludes: false
|
||||
AttributeMacros:
|
||||
- EIGEN_STRONG_INLINE
|
||||
|
||||
@@ -176,35 +176,28 @@ class AccelerateImpl : public SparseSolverBase<AccelerateImpl<MatrixType_, UpLo_
|
||||
AccelerateImpl() {
|
||||
m_isInitialized = false;
|
||||
|
||||
EIGEN_IF_CONSTEXPR((UpLo_ & Symmetric) == Symmetric) {
|
||||
EIGEN_IF_CONSTEXPR ((UpLo_ & Symmetric) == Symmetric) {
|
||||
m_sparseKind = SparseSymmetric;
|
||||
m_triType = (UpLo_ & Lower) ? SparseLowerTriangle : SparseUpperTriangle;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR((UpLo_ & UnitLower) == UnitLower) {
|
||||
} else EIGEN_IF_CONSTEXPR ((UpLo_ & UnitLower) == UnitLower) {
|
||||
m_sparseKind = SparseUnitTriangular;
|
||||
m_triType = SparseLowerTriangle;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR((UpLo_ & UnitUpper) == UnitUpper) {
|
||||
} else EIGEN_IF_CONSTEXPR ((UpLo_ & UnitUpper) == UnitUpper) {
|
||||
m_sparseKind = SparseUnitTriangular;
|
||||
m_triType = SparseUpperTriangle;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR((UpLo_ & StrictlyLower) == StrictlyLower) {
|
||||
} else EIGEN_IF_CONSTEXPR ((UpLo_ & StrictlyLower) == StrictlyLower) {
|
||||
m_sparseKind = SparseTriangular;
|
||||
m_triType = SparseLowerTriangle;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR((UpLo_ & StrictlyUpper) == StrictlyUpper) {
|
||||
} else EIGEN_IF_CONSTEXPR ((UpLo_ & StrictlyUpper) == StrictlyUpper) {
|
||||
m_sparseKind = SparseTriangular;
|
||||
m_triType = SparseUpperTriangle;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR((UpLo_ & Lower) == Lower) {
|
||||
} else EIGEN_IF_CONSTEXPR ((UpLo_ & Lower) == Lower) {
|
||||
m_sparseKind = SparseTriangular;
|
||||
m_triType = SparseLowerTriangle;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR((UpLo_ & Upper) == Upper) {
|
||||
} else EIGEN_IF_CONSTEXPR ((UpLo_ & Upper) == Upper) {
|
||||
m_sparseKind = SparseTriangular;
|
||||
m_triType = SparseUpperTriangle;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_sparseKind = SparseOrdinary;
|
||||
m_triType = (UpLo_ & Lower) ? SparseLowerTriangle : SparseUpperTriangle;
|
||||
}
|
||||
|
||||
@@ -332,8 +332,8 @@ struct ldlt_inplace<Lower> {
|
||||
mat.coeffRef(i, k) = numext::conj(mat.coeffRef(index_of_biggest_in_corner, i));
|
||||
mat.coeffRef(index_of_biggest_in_corner, i) = numext::conj(tmp);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex)
|
||||
mat.coeffRef(index_of_biggest_in_corner, k) = numext::conj(mat.coeff(index_of_biggest_in_corner, k));
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex)
|
||||
mat.coeffRef(index_of_biggest_in_corner, k) = numext::conj(mat.coeff(index_of_biggest_in_corner, k));
|
||||
}
|
||||
|
||||
// partition the matrix:
|
||||
|
||||
@@ -83,11 +83,11 @@ cholmod_sparse viewAsCholmod(Ref<SparseMatrix<Scalar_, Options_, StorageIndex_>
|
||||
res.dtype = 0;
|
||||
res.stype = -1;
|
||||
|
||||
EIGEN_IF_CONSTEXPR((std::is_same<StorageIndex_, int>::value)) { res.itype = CHOLMOD_INT; }
|
||||
else EIGEN_IF_CONSTEXPR((std::is_same<StorageIndex_, SuiteSparse_long>::value)) {
|
||||
EIGEN_IF_CONSTEXPR ((std::is_same<StorageIndex_, int>::value)) {
|
||||
res.itype = CHOLMOD_INT;
|
||||
} else EIGEN_IF_CONSTEXPR ((std::is_same<StorageIndex_, SuiteSparse_long>::value)) {
|
||||
res.itype = CHOLMOD_LONG;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
eigen_assert(false && "Index type not supported yet");
|
||||
}
|
||||
|
||||
|
||||
+12
-16
@@ -115,10 +115,9 @@ class Concat : public internal::dense_xpr_base<Concat<Direction, LhsType, RhsTyp
|
||||
int(RhsType::ColsAtCompileTime) == Dynamic ||
|
||||
int(LhsType::ColsAtCompileTime) == int(RhsType::ColsAtCompileTime),
|
||||
YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES)
|
||||
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
|
||||
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
|
||||
eigen_assert(lhs.cols() == rhs.cols() && "vcat: number of columns must match");
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
eigen_assert(lhs.rows() == rhs.rows() && "hcat: number of rows must match");
|
||||
}
|
||||
}
|
||||
@@ -175,13 +174,12 @@ struct evaluator<Concat<Direction, LhsType, RhsType>> : evaluator_base<Concat<Di
|
||||
m_lhsCols(xpr.lhs().cols()) {}
|
||||
|
||||
EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
|
||||
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
|
||||
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
|
||||
if (row < m_lhsRows.value())
|
||||
return m_lhsImpl.coeff(row, col);
|
||||
else
|
||||
return m_rhsImpl.coeff(row - m_lhsRows.value(), col);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
if (col < m_lhsCols.value())
|
||||
return m_lhsImpl.coeff(row, col);
|
||||
else
|
||||
@@ -200,22 +198,21 @@ struct evaluator<Concat<Direction, LhsType, RhsType>> : evaluator_base<Concat<Di
|
||||
template <int LoadMode, typename PacketType>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
|
||||
constexpr int packetSize = unpacket_traits<PacketType>::size;
|
||||
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
|
||||
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
|
||||
const Index boundary = m_lhsRows.value();
|
||||
if (row >= boundary) return m_rhsImpl.template packet<LoadMode, PacketType>(row - boundary, col);
|
||||
// Column-major: inner=rows, packet extends along rows and may straddle the row boundary.
|
||||
// Row-major: inner=cols, packet extends along cols — never crosses the row boundary.
|
||||
EIGEN_IF_CONSTEXPR(!IsRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (!IsRowMajor) {
|
||||
if (row + packetSize > boundary) return packetBoundary<LoadMode, PacketType>(row, col);
|
||||
}
|
||||
return m_lhsImpl.template packet<LoadMode, PacketType>(row, col);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const Index boundary = m_lhsCols.value();
|
||||
if (col >= boundary) return m_rhsImpl.template packet<LoadMode, PacketType>(row, col - boundary);
|
||||
// Row-major: inner=cols, packet extends along cols and may straddle the col boundary.
|
||||
// Column-major: inner=rows, packet extends along rows — never crosses the col boundary.
|
||||
EIGEN_IF_CONSTEXPR(IsRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (IsRowMajor) {
|
||||
if (col + packetSize > boundary) return packetBoundary<LoadMode, PacketType>(row, col);
|
||||
}
|
||||
return m_lhsImpl.template packet<LoadMode, PacketType>(row, col);
|
||||
@@ -233,20 +230,19 @@ struct evaluator<Concat<Direction, LhsType, RhsType>> : evaluator_base<Concat<Di
|
||||
|
||||
template <int LoadMode, typename PacketType>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index row, Index col, Index begin, Index count) const {
|
||||
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
|
||||
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
|
||||
const Index boundary = m_lhsRows.value();
|
||||
if (row >= boundary)
|
||||
return m_rhsImpl.template packetSegment<LoadMode, PacketType>(row - boundary, col, begin, count);
|
||||
EIGEN_IF_CONSTEXPR(!IsRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (!IsRowMajor) {
|
||||
if (row + begin + count > boundary) return packetSegmentBoundary<LoadMode, PacketType>(row, col, begin, count);
|
||||
}
|
||||
return m_lhsImpl.template packetSegment<LoadMode, PacketType>(row, col, begin, count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const Index boundary = m_lhsCols.value();
|
||||
if (col >= boundary)
|
||||
return m_rhsImpl.template packetSegment<LoadMode, PacketType>(row, col - boundary, begin, count);
|
||||
EIGEN_IF_CONSTEXPR(IsRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (IsRowMajor) {
|
||||
if (col + begin + count > boundary) return packetSegmentBoundary<LoadMode, PacketType>(row, col, begin, count);
|
||||
}
|
||||
return m_lhsImpl.template packetSegment<LoadMode, PacketType>(row, col, begin, count);
|
||||
|
||||
@@ -1429,9 +1429,10 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
|
||||
|
||||
template <int LoadMode, typename PacketType>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const {
|
||||
EIGEN_IF_CONSTEXPR(ForwardLinearAccess)
|
||||
return m_argImpl.template packet<LoadMode, PacketType>(m_linear_offset.value() + index);
|
||||
else return packet<LoadMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0);
|
||||
EIGEN_IF_CONSTEXPR (ForwardLinearAccess)
|
||||
return m_argImpl.template packet<LoadMode, PacketType>(m_linear_offset.value() + index);
|
||||
else
|
||||
return packet<LoadMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0);
|
||||
}
|
||||
|
||||
template <int StoreMode, typename PacketType>
|
||||
@@ -1441,10 +1442,11 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
|
||||
|
||||
template <int StoreMode, typename PacketType>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) {
|
||||
EIGEN_IF_CONSTEXPR(ForwardLinearAccess)
|
||||
return m_argImpl.template writePacket<StoreMode, PacketType>(m_linear_offset.value() + index, x);
|
||||
else return writePacket<StoreMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index,
|
||||
RowsAtCompileTime == 1 ? index : 0, x);
|
||||
EIGEN_IF_CONSTEXPR (ForwardLinearAccess)
|
||||
return m_argImpl.template writePacket<StoreMode, PacketType>(m_linear_offset.value() + index, x);
|
||||
else
|
||||
return writePacket<StoreMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0,
|
||||
x);
|
||||
}
|
||||
|
||||
template <int LoadMode, typename PacketType>
|
||||
@@ -1455,10 +1457,11 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
|
||||
|
||||
template <int LoadMode, typename PacketType>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index index, Index begin, Index count) const {
|
||||
EIGEN_IF_CONSTEXPR(ForwardLinearAccess)
|
||||
return m_argImpl.template packetSegment<LoadMode, PacketType>(m_linear_offset.value() + index, begin, count);
|
||||
else return packetSegment<LoadMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index,
|
||||
RowsAtCompileTime == 1 ? index : 0, begin, count);
|
||||
EIGEN_IF_CONSTEXPR (ForwardLinearAccess)
|
||||
return m_argImpl.template packetSegment<LoadMode, PacketType>(m_linear_offset.value() + index, begin, count);
|
||||
else
|
||||
return packetSegment<LoadMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0,
|
||||
begin, count);
|
||||
}
|
||||
|
||||
template <int StoreMode, typename PacketType>
|
||||
@@ -1471,11 +1474,12 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
|
||||
template <int StoreMode, typename PacketType>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacketSegment(Index index, const PacketType& x, Index begin,
|
||||
Index count) {
|
||||
EIGEN_IF_CONSTEXPR(ForwardLinearAccess)
|
||||
return m_argImpl.template writePacketSegment<StoreMode, PacketType>(m_linear_offset.value() + index, x, begin,
|
||||
count);
|
||||
else return writePacketSegment<StoreMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index,
|
||||
RowsAtCompileTime == 1 ? index : 0, x, begin, count);
|
||||
EIGEN_IF_CONSTEXPR (ForwardLinearAccess)
|
||||
return m_argImpl.template writePacketSegment<StoreMode, PacketType>(m_linear_offset.value() + index, x, begin,
|
||||
count);
|
||||
else
|
||||
return writePacketSegment<StoreMode, PacketType>(RowsAtCompileTime == 1 ? 0 : index,
|
||||
RowsAtCompileTime == 1 ? index : 0, x, begin, count);
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
@@ -142,8 +142,9 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
||||
#endif
|
||||
DenseBase<Derived>::NullaryExpr(Index size, const CustomNullaryOp& func) {
|
||||
EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived)
|
||||
EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { return CwiseNullaryOp<CustomNullaryOp, PlainObject>(1, size, func); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (RowsAtCompileTime == 1) {
|
||||
return CwiseNullaryOp<CustomNullaryOp, PlainObject>(1, size, func);
|
||||
} else {
|
||||
return CwiseNullaryOp<CustomNullaryOp, PlainObject>(size, 1, func);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,30 +122,26 @@ class CwiseTernaryOp : public CwiseTernaryOpImpl<TernaryOp, Arg1Type, Arg2Type,
|
||||
EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Index rows() const {
|
||||
// return the fixed size type if available to enable compile time
|
||||
// optimizations
|
||||
EIGEN_IF_CONSTEXPR(internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
|
||||
internal::traits<internal::remove_all_t<Arg2Nested>>::RowsAtCompileTime == Dynamic) {
|
||||
EIGEN_IF_CONSTEXPR (internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
|
||||
internal::traits<internal::remove_all_t<Arg2Nested>>::RowsAtCompileTime == Dynamic) {
|
||||
return m_arg3.rows();
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
|
||||
internal::traits<internal::remove_all_t<Arg3Nested>>::RowsAtCompileTime == Dynamic) {
|
||||
} else EIGEN_IF_CONSTEXPR (internal::traits<internal::remove_all_t<Arg1Nested>>::RowsAtCompileTime == Dynamic &&
|
||||
internal::traits<internal::remove_all_t<Arg3Nested>>::RowsAtCompileTime == Dynamic) {
|
||||
return m_arg2.rows();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return m_arg1.rows();
|
||||
}
|
||||
}
|
||||
EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Index cols() const {
|
||||
// return the fixed size type if available to enable compile time
|
||||
// optimizations
|
||||
EIGEN_IF_CONSTEXPR(internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
|
||||
internal::traits<internal::remove_all_t<Arg2Nested>>::ColsAtCompileTime == Dynamic) {
|
||||
EIGEN_IF_CONSTEXPR (internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
|
||||
internal::traits<internal::remove_all_t<Arg2Nested>>::ColsAtCompileTime == Dynamic) {
|
||||
return m_arg3.cols();
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
|
||||
internal::traits<internal::remove_all_t<Arg3Nested>>::ColsAtCompileTime == Dynamic) {
|
||||
} else EIGEN_IF_CONSTEXPR (internal::traits<internal::remove_all_t<Arg1Nested>>::ColsAtCompileTime == Dynamic &&
|
||||
internal::traits<internal::remove_all_t<Arg3Nested>>::ColsAtCompileTime == Dynamic) {
|
||||
return m_arg2.cols();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return m_arg1.cols();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1448,8 +1448,9 @@ inline void pstore1(typename unpacket_traits<Packet>::type* to, const typename u
|
||||
* The pointer \a from must be aligned on a \a Alignment bytes boundary. */
|
||||
template <typename Packet, int Alignment>
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_traits<Packet>::type* from) {
|
||||
EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits<Packet>::alignment) { return pload<Packet>(from); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
|
||||
return pload<Packet>(from);
|
||||
} else {
|
||||
return ploadu<Packet>(from);
|
||||
}
|
||||
}
|
||||
@@ -1459,8 +1460,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_trai
|
||||
template <typename Packet, int Alignment>
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(const typename unpacket_traits<Packet>::type* from,
|
||||
const Index n, const Index offset = 0) {
|
||||
EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits<Packet>::alignment) { return pload_partial<Packet>(from, n, offset); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
|
||||
return pload_partial<Packet>(from, n, offset);
|
||||
} else {
|
||||
return ploadu_partial<Packet>(from, n, offset);
|
||||
}
|
||||
}
|
||||
@@ -1469,8 +1471,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(const typename unpac
|
||||
* The pointer \a from must be aligned on a \a Alignment bytes boundary. */
|
||||
template <typename Scalar, typename Packet, int Alignment>
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& from) {
|
||||
EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits<Packet>::alignment) { pstore(to, from); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
|
||||
pstore(to, from);
|
||||
} else {
|
||||
pstoreu(to, from);
|
||||
}
|
||||
}
|
||||
@@ -1480,8 +1483,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& fro
|
||||
template <typename Scalar, typename Packet, int Alignment>
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret_partial(Scalar* to, const Packet& from, const Index n,
|
||||
const Index offset = 0) {
|
||||
EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits<Packet>::alignment) { pstore_partial(to, from, n, offset); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
|
||||
pstore_partial(to, from, n, offset);
|
||||
} else {
|
||||
pstoreu_partial(to, from, n, offset);
|
||||
}
|
||||
}
|
||||
@@ -1677,8 +1681,9 @@ template <typename Packet, int Alignment>
|
||||
EIGEN_DEVICE_FUNC inline Packet ploadtSegment(const typename unpacket_traits<Packet>::type* from, Index begin,
|
||||
Index count) {
|
||||
constexpr int RequiredAlignment = unpacket_traits<Packet>::alignment;
|
||||
EIGEN_IF_CONSTEXPR(Alignment >= RequiredAlignment) { return ploadSegment<Packet>(from, begin, count); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) {
|
||||
return ploadSegment<Packet>(from, begin, count);
|
||||
} else {
|
||||
return ploaduSegment<Packet>(from, begin, count);
|
||||
}
|
||||
}
|
||||
@@ -1688,8 +1693,9 @@ Elements outside of the range [begin, begin + count) are not defined.*/
|
||||
template <typename Scalar, typename Packet, int Alignment>
|
||||
EIGEN_DEVICE_FUNC inline void pstoretSegment(Scalar* to, const Packet& from, Index begin, Index count) {
|
||||
constexpr int RequiredAlignment = unpacket_traits<Packet>::alignment;
|
||||
EIGEN_IF_CONSTEXPR(Alignment >= RequiredAlignment) { pstoreSegment<Scalar, Packet>(to, from, begin, count); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) {
|
||||
pstoreSegment<Scalar, Packet>(to, from, begin, count);
|
||||
} else {
|
||||
pstoreuSegment<Scalar, Packet>(to, from, begin, count);
|
||||
}
|
||||
}
|
||||
|
||||
+3
-2
@@ -150,8 +150,9 @@ std::ostream& print_matrix(std::ostream& s, const Derived& _m, const IOFormat& f
|
||||
if (fmt.precision == StreamPrecision) {
|
||||
explicit_precision = 0;
|
||||
} else if (fmt.precision == FullPrecision) {
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsInteger) { explicit_precision = 0; }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsInteger) {
|
||||
explicit_precision = 0;
|
||||
} else {
|
||||
explicit_precision = significant_decimals_impl<Scalar>::run();
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -199,13 +199,13 @@ class IndexedViewImpl<XprType, RowIndices, ColIndices, StorageKind, true>
|
||||
IndexedViewImpl(XprType& xpr, const T0& rowIndices, const T1& colIndices) : Base(xpr, rowIndices, colIndices) {}
|
||||
|
||||
Index rowIncrement() const {
|
||||
EIGEN_IF_CONSTEXPR(traits<Derived>::RowIncr != DynamicIndex && traits<Derived>::RowIncr != Undefined) {
|
||||
EIGEN_IF_CONSTEXPR (traits<Derived>::RowIncr != DynamicIndex && traits<Derived>::RowIncr != Undefined) {
|
||||
return traits<Derived>::RowIncr;
|
||||
}
|
||||
return IndexedViewHelper<RowIndices>::incr(this->rowIndices());
|
||||
}
|
||||
Index colIncrement() const {
|
||||
EIGEN_IF_CONSTEXPR(traits<Derived>::ColIncr != DynamicIndex && traits<Derived>::ColIncr != Undefined) {
|
||||
EIGEN_IF_CONSTEXPR (traits<Derived>::ColIncr != DynamicIndex && traits<Derived>::ColIncr != Undefined) {
|
||||
return traits<Derived>::ColIncr;
|
||||
}
|
||||
return IndexedViewHelper<ColIndices>::incr(this->colIndices());
|
||||
@@ -228,14 +228,14 @@ class IndexedViewImpl<XprType, RowIndices, ColIndices, StorageKind, true>
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC constexpr Index innerStride() const noexcept {
|
||||
EIGEN_IF_CONSTEXPR(traits<Derived>::InnerStrideAtCompileTime != Dynamic) {
|
||||
EIGEN_IF_CONSTEXPR (traits<Derived>::InnerStrideAtCompileTime != Dynamic) {
|
||||
return traits<Derived>::InnerStrideAtCompileTime;
|
||||
}
|
||||
return innerIncrement() * this->nestedExpression().innerStride();
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC constexpr Index outerStride() const noexcept {
|
||||
EIGEN_IF_CONSTEXPR(traits<Derived>::OuterStrideAtCompileTime != Dynamic) {
|
||||
EIGEN_IF_CONSTEXPR (traits<Derived>::OuterStrideAtCompileTime != Dynamic) {
|
||||
return traits<Derived>::OuterStrideAtCompileTime;
|
||||
}
|
||||
return outerIncrement() * this->nestedExpression().outerStride();
|
||||
|
||||
@@ -921,7 +921,7 @@ struct copysign_impl<Scalar, true, IsInteger> {
|
||||
template <typename Scalar>
|
||||
struct copysign_impl<Scalar, false, true> {
|
||||
EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& a, const Scalar& b) {
|
||||
EIGEN_IF_CONSTEXPR(!NumTraits<Scalar>::IsSigned) return a;
|
||||
EIGEN_IF_CONSTEXPR (!NumTraits<Scalar>::IsSigned) return a;
|
||||
const Scalar abs_a = a < Scalar(0) ? -a : a;
|
||||
return b < Scalar(0) ? -abs_a : abs_a;
|
||||
}
|
||||
|
||||
@@ -203,7 +203,7 @@ struct evaluator<PartialReduxExpr<ArgType, MemberOp, Direction> >
|
||||
// Workaround for issue 1612 (closed): when PacketSize==1 (i.e. complex<double> with 128bits registers) the
|
||||
// storage-order of panel gets reversed and methods like packetByOuterInner do not make sense in this context, so
|
||||
// bypass "vectorization":
|
||||
EIGEN_IF_CONSTEXPR(PacketSize == 1) return internal::pset1<PacketType>(coeff(idx));
|
||||
EIGEN_IF_CONSTEXPR (PacketSize == 1) return internal::pset1<PacketType>(coeff(idx));
|
||||
|
||||
Index startRow = Direction == Vertical ? 0 : idx;
|
||||
Index startCol = Direction == Vertical ? idx : 0;
|
||||
|
||||
@@ -171,8 +171,9 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
|
||||
*
|
||||
* See DenseCoeffsBase<Derived,ReadOnlyAccessors>::coeff(Index) const for details. */
|
||||
EIGEN_DEVICE_FUNC constexpr const Scalar& coeff(Index rowId, Index colId) const {
|
||||
EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; }
|
||||
else { // column-major
|
||||
EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) {
|
||||
return m_storage.data()[colId + rowId * m_storage.cols()];
|
||||
} else { // column-major
|
||||
return m_storage.data()[rowId + colId * m_storage.rows()];
|
||||
}
|
||||
}
|
||||
@@ -188,8 +189,9 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
|
||||
*
|
||||
* See DenseCoeffsBase<Derived,WriteAccessors>::coeffRef(Index,Index) const for details. */
|
||||
EIGEN_DEVICE_FUNC constexpr Scalar& coeffRef(Index rowId, Index colId) {
|
||||
EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; }
|
||||
else { // column-major
|
||||
EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) {
|
||||
return m_storage.data()[colId + rowId * m_storage.cols()];
|
||||
} else { // column-major
|
||||
return m_storage.data()[rowId + colId * m_storage.rows()];
|
||||
}
|
||||
}
|
||||
@@ -203,8 +205,9 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
|
||||
/** This is the const version of coeffRef(Index,Index) which is thus synonym of coeff(Index,Index).
|
||||
* It is provided for convenience. */
|
||||
EIGEN_DEVICE_FUNC constexpr const Scalar& coeffRef(Index rowId, Index colId) const {
|
||||
EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; }
|
||||
else { // column-major
|
||||
EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) {
|
||||
return m_storage.data()[colId + rowId * m_storage.cols()];
|
||||
} else { // column-major
|
||||
return m_storage.data()[rowId + colId * m_storage.rows()];
|
||||
}
|
||||
}
|
||||
@@ -303,8 +306,9 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
|
||||
#ifdef EIGEN_INITIALIZE_COEFFS
|
||||
bool size_changed = size != this->size();
|
||||
#endif
|
||||
EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { m_storage.resize(size, 1, size); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (RowsAtCompileTime == 1) {
|
||||
m_storage.resize(size, 1, size);
|
||||
} else {
|
||||
m_storage.resize(size, size, 1);
|
||||
}
|
||||
#ifdef EIGEN_INITIALIZE_COEFFS
|
||||
@@ -347,15 +351,13 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type {
|
||||
other.rows(), other.cols());
|
||||
#endif
|
||||
const Index othersize = other.rows() * other.cols();
|
||||
EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) {
|
||||
EIGEN_IF_CONSTEXPR (RowsAtCompileTime == 1) {
|
||||
eigen_assert(other.rows() == 1 || other.cols() == 1);
|
||||
resize(1, othersize);
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(ColsAtCompileTime == 1) {
|
||||
} else EIGEN_IF_CONSTEXPR (ColsAtCompileTime == 1) {
|
||||
eigen_assert(other.rows() == 1 || other.cols() == 1);
|
||||
resize(othersize, 1);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
resize(other.rows(), other.cols());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -899,16 +899,15 @@ struct triangular_diagonal_product_impl {
|
||||
const Index rows = matrix.rows();
|
||||
const Index cols = matrix.cols();
|
||||
for (Index col = 0; col < cols; ++col) {
|
||||
EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) {
|
||||
EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) {
|
||||
const Index end = (std::min)(rows, ((Mode & (UnitDiag | ZeroDiag)) ? col : col + 1));
|
||||
addStoredSegment(dst, matrix, diagonal, 0, end, col, alpha);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const Index begin = ((Mode & (UnitDiag | ZeroDiag)) ? col + 1 : col);
|
||||
addStoredSegment(dst, matrix, diagonal, begin, rows - begin, col, alpha);
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) {
|
||||
if (col < rows) addUnitCoeff(dst, diagonal, col, alpha);
|
||||
}
|
||||
}
|
||||
@@ -928,10 +927,9 @@ struct triangular_diagonal_product_impl {
|
||||
template <typename Dest, typename Alpha>
|
||||
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addUnitCoeff(Dest& dst, const DiagonalType& diagonal, Index index,
|
||||
const Alpha& alpha) {
|
||||
EIGEN_IF_CONSTEXPR(ProductOrder == OnTheLeft) {
|
||||
EIGEN_IF_CONSTEXPR (ProductOrder == OnTheLeft) {
|
||||
dst.coeffRef(index, index) += alpha * (diagonal.coeff(index) * MatrixScalar(1));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
dst.coeffRef(index, index) += alpha * (MatrixScalar(1) * diagonal.coeff(index));
|
||||
}
|
||||
}
|
||||
@@ -1041,10 +1039,9 @@ struct selfadjoint_diagonal_product_impl {
|
||||
|
||||
// Stored half: one column-strided segment per output column.
|
||||
for (Index col = 0; col < size; ++col) {
|
||||
EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) {
|
||||
EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) {
|
||||
storedSegment<Accumulate>(dst, matrix, diagonal, 0, col + 1, col, alpha);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
storedSegment<Accumulate>(dst, matrix, diagonal, col, size - col, col, alpha);
|
||||
}
|
||||
}
|
||||
@@ -1053,7 +1050,7 @@ struct selfadjoint_diagonal_product_impl {
|
||||
for (Index ib = 0; ib < size; ib += BlockSize) {
|
||||
const Index ib_end = numext::mini(size, ib + BlockSize);
|
||||
const Index br = ib_end - ib;
|
||||
EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) {
|
||||
EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) {
|
||||
// Off-diagonal: write strict-lower of dst from strict-upper of source.
|
||||
for (Index jb = 0; jb < ib; jb += BlockSize) {
|
||||
const Index bc = numext::mini(jb + BlockSize, ib) - jb;
|
||||
@@ -1062,8 +1059,7 @@ struct selfadjoint_diagonal_product_impl {
|
||||
// Diagonal tile: in-tile strict-lower mirror.
|
||||
for (Index col = ib; col < ib_end; ++col)
|
||||
conjugateSegment<Accumulate>(dst, matrix, diagonal, col + 1, ib_end - col - 1, col, alpha);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// Off-diagonal: write strict-upper of dst from strict-lower of source.
|
||||
for (Index jb = ib_end; jb < size; jb += BlockSize) {
|
||||
const Index bc = numext::mini(size, jb + BlockSize) - jb;
|
||||
@@ -1083,10 +1079,9 @@ struct selfadjoint_diagonal_product_impl {
|
||||
if (size <= 0) return;
|
||||
auto dstSegment = dst.col(col).segment(begin, size);
|
||||
auto srcSegment = matrix.col(col).segment(begin, size);
|
||||
EIGEN_IF_CONSTEXPR(Accumulate) {
|
||||
EIGEN_IF_CONSTEXPR (Accumulate) {
|
||||
diagonal_product_segment_impl<ProductOrder>::run(dstSegment, srcSegment, diagonal, begin, col, alpha);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
diagonal_product_segment_impl<ProductOrder>::runOverwrite(dstSegment, srcSegment, diagonal, begin, col);
|
||||
}
|
||||
}
|
||||
@@ -1098,10 +1093,9 @@ struct selfadjoint_diagonal_product_impl {
|
||||
if (size <= 0) return;
|
||||
auto dstSegment = dst.col(col).segment(begin, size);
|
||||
auto srcSegment = matrix.row(col).segment(begin, size).conjugate().transpose();
|
||||
EIGEN_IF_CONSTEXPR(Accumulate) {
|
||||
EIGEN_IF_CONSTEXPR (Accumulate) {
|
||||
diagonal_product_segment_impl<ProductOrder>::run(dstSegment, srcSegment, diagonal, begin, col, alpha);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
diagonal_product_segment_impl<ProductOrder>::runOverwrite(dstSegment, srcSegment, diagonal, begin, col);
|
||||
}
|
||||
}
|
||||
@@ -1115,17 +1109,18 @@ struct selfadjoint_diagonal_product_impl {
|
||||
Index br, Index bc, const Alpha& alpha) {
|
||||
auto dstBlock = dst.block(ib, jb, br, bc);
|
||||
auto srcAdjoint = matrix.block(jb, ib, bc, br).adjoint();
|
||||
EIGEN_IF_CONSTEXPR(ProductOrder == OnTheRight) {
|
||||
EIGEN_IF_CONSTEXPR (ProductOrder == OnTheRight) {
|
||||
auto scaled = srcAdjoint * diagonal.segment(jb, bc).asDiagonal();
|
||||
EIGEN_IF_CONSTEXPR(Accumulate) { dstBlock.noalias() += alpha * scaled; }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Accumulate) {
|
||||
dstBlock.noalias() += alpha * scaled;
|
||||
} else {
|
||||
dstBlock.noalias() = scaled;
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
auto scaled = diagonal.segment(ib, br).asDiagonal() * srcAdjoint;
|
||||
EIGEN_IF_CONSTEXPR(Accumulate) { dstBlock.noalias() += alpha * scaled; }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Accumulate) {
|
||||
dstBlock.noalias() += alpha * scaled;
|
||||
} else {
|
||||
dstBlock.noalias() = scaled;
|
||||
}
|
||||
}
|
||||
@@ -1180,13 +1175,12 @@ struct generic_product_impl<Lhs, Rhs, SelfAdjointShape, DiagonalShape, ProductTa
|
||||
|
||||
template <typename Dest>
|
||||
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
|
||||
EIGEN_IF_CONSTEXPR(LhsBlasTraits::HasScalarFactor) {
|
||||
EIGEN_IF_CONSTEXPR (LhsBlasTraits::HasScalarFactor) {
|
||||
// Folded scalar factor present: zero dst then accumulate at the extracted alpha.
|
||||
Scalar factor = LhsBlasTraits::extractScalarFactor(lhs.nestedExpression());
|
||||
dst.setZero();
|
||||
Kernel::run(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal(), factor);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// No scalar factor: kernel writes every entry exactly once, skip setZero.
|
||||
Kernel::runOverwrite(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal());
|
||||
}
|
||||
@@ -1219,12 +1213,11 @@ struct generic_product_impl<Lhs, Rhs, DiagonalShape, SelfAdjointShape, ProductTa
|
||||
|
||||
template <typename Dest>
|
||||
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
|
||||
EIGEN_IF_CONSTEXPR(RhsBlasTraits::HasScalarFactor) {
|
||||
EIGEN_IF_CONSTEXPR (RhsBlasTraits::HasScalarFactor) {
|
||||
Scalar factor = RhsBlasTraits::extractScalarFactor(rhs.nestedExpression());
|
||||
dst.setZero();
|
||||
Kernel::run(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal(), factor);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
Kernel::runOverwrite(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal());
|
||||
}
|
||||
}
|
||||
@@ -1471,11 +1464,11 @@ struct triangular_diagonal_product_lazy_evaluator_base : evaluator_base<Derived>
|
||||
const bool inActive = ((Mode & Upper) == Upper) ? (row <= col) : (row >= col);
|
||||
if (!inActive) return Scalar(0);
|
||||
if (row == col) {
|
||||
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) {
|
||||
return ProductOrder == OnTheLeft ? Scalar(m_diagImpl.coeff(row) * MatrixScalar(1))
|
||||
: Scalar(MatrixScalar(1) * m_diagImpl.coeff(col));
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag) return Scalar(0);
|
||||
EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag) return Scalar(0);
|
||||
}
|
||||
return ProductOrder == OnTheLeft ? Scalar(m_diagImpl.coeff(row) * m_matImpl.coeff(row, col))
|
||||
: Scalar(m_matImpl.coeff(row, col) * m_diagImpl.coeff(col));
|
||||
@@ -1701,8 +1694,9 @@ struct transposition_matrix_product {
|
||||
|
||||
for (Index k = (Transposed ? size - 1 : 0); Transposed ? k >= 0 : k < size; Transposed ? --k : ++k)
|
||||
if (Index(j = tr.coeff(k)) != k) {
|
||||
EIGEN_IF_CONSTEXPR(Side == OnTheLeft) { dst.row(k).swap(dst.row(j)); }
|
||||
else EIGEN_IF_CONSTEXPR(Side == OnTheRight) {
|
||||
EIGEN_IF_CONSTEXPR (Side == OnTheLeft) {
|
||||
dst.row(k).swap(dst.row(j));
|
||||
} else EIGEN_IF_CONSTEXPR (Side == OnTheRight) {
|
||||
dst.col(k).swap(dst.col(j));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,12 +126,11 @@ class RefBase : public MapBase<Derived> {
|
||||
// Determine runtime rows and columns.
|
||||
Index rows = expr.rows();
|
||||
Index cols = expr.cols();
|
||||
EIGEN_IF_CONSTEXPR(PlainObjectType::RowsAtCompileTime == 1) {
|
||||
EIGEN_IF_CONSTEXPR (PlainObjectType::RowsAtCompileTime == 1) {
|
||||
eigen_assert(expr.rows() == 1 || expr.cols() == 1);
|
||||
rows = 1;
|
||||
cols = expr.size();
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(PlainObjectType::ColsAtCompileTime == 1) {
|
||||
} else EIGEN_IF_CONSTEXPR (PlainObjectType::ColsAtCompileTime == 1) {
|
||||
eigen_assert(expr.rows() == 1 || expr.cols() == 1);
|
||||
rows = expr.size();
|
||||
cols = 1;
|
||||
|
||||
@@ -296,11 +296,10 @@ struct reshaped_evaluator<ArgType, Rows, Cols, Order, /* HasDirectAccess */ fals
|
||||
typedef std::pair<Index, Index> RowCol;
|
||||
|
||||
EIGEN_DEVICE_FUNC constexpr inline RowCol index_remap(Index rowId, Index colId) const {
|
||||
EIGEN_IF_CONSTEXPR(Order == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Order == ColMajor) {
|
||||
const Index nth_elem_idx = colId * m_xpr.rows() + rowId;
|
||||
return RowCol(nth_elem_idx % m_xpr.nestedExpression().rows(), nth_elem_idx / m_xpr.nestedExpression().rows());
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const Index nth_elem_idx = colId + rowId * m_xpr.cols();
|
||||
return RowCol(nth_elem_idx / m_xpr.nestedExpression().cols(), nth_elem_idx % m_xpr.nestedExpression().cols());
|
||||
}
|
||||
|
||||
@@ -219,11 +219,10 @@ class SelfAdjointView : public TriangularBase<SelfAdjointView<MatrixType_, UpLo>
|
||||
const Index n = m_matrix.rows();
|
||||
for (Index col = 0; col < n; ++col) {
|
||||
RealScalar_ abs_col_sum;
|
||||
EIGEN_IF_CONSTEXPR(UpLo == Lower) {
|
||||
EIGEN_IF_CONSTEXPR (UpLo == Lower) {
|
||||
abs_col_sum =
|
||||
m_matrix.col(col).tail(n - col).template lpNorm<1>() + m_matrix.row(col).head(col).template lpNorm<1>();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
abs_col_sum =
|
||||
m_matrix.col(col).head(col).template lpNorm<1>() + m_matrix.row(col).tail(n - col).template lpNorm<1>();
|
||||
}
|
||||
|
||||
@@ -391,11 +391,11 @@ class TriangularView
|
||||
/** \returns the determinant of the triangular matrix
|
||||
* \sa MatrixBase::determinant() */
|
||||
EIGEN_DEVICE_FUNC Scalar determinant() const {
|
||||
EIGEN_IF_CONSTEXPR(Mode & UnitDiag) { return 1; }
|
||||
else EIGEN_IF_CONSTEXPR(Mode & ZeroDiag) {
|
||||
EIGEN_IF_CONSTEXPR (Mode & UnitDiag) {
|
||||
return 1;
|
||||
} else EIGEN_IF_CONSTEXPR (Mode & ZeroDiag) {
|
||||
return 0;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return m_matrix.diagonal().prod();
|
||||
}
|
||||
}
|
||||
@@ -777,18 +777,20 @@ class triangular_dense_assignment_kernel
|
||||
#endif
|
||||
|
||||
EIGEN_DEVICE_FUNC void assignDiagonalCoeff(Index id) {
|
||||
EIGEN_IF_CONSTEXPR(Mode == UnitDiag && SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(1)); }
|
||||
else EIGEN_IF_CONSTEXPR(Mode == ZeroDiag && SetOpposite) {
|
||||
EIGEN_IF_CONSTEXPR (Mode == UnitDiag && SetOpposite) {
|
||||
m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(1));
|
||||
} else EIGEN_IF_CONSTEXPR (Mode == ZeroDiag && SetOpposite) {
|
||||
m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(0));
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(Mode == 0) {
|
||||
} else EIGEN_IF_CONSTEXPR (Mode == 0) {
|
||||
Base::assignCoeff(id, id);
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC void assignOppositeCoeff(Index row, Index col) {
|
||||
eigen_internal_assert(row != col);
|
||||
EIGEN_IF_CONSTEXPR(SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(row, col), Scalar(0)); }
|
||||
EIGEN_IF_CONSTEXPR (SetOpposite) {
|
||||
m_functor.assignCoeff(m_dst.coeffRef(row, col), Scalar(0));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -894,8 +896,9 @@ struct triangular_assignment_loop {
|
||||
kernel.assignDiagonalCoeff(row);
|
||||
else if (((Mode & Lower) && row > col) || ((Mode & Upper) && row < col))
|
||||
kernel.assignCoeff(row, col);
|
||||
else
|
||||
EIGEN_IF_CONSTEXPR(SetOpposite) { kernel.assignOppositeCoeff(row, col); }
|
||||
else EIGEN_IF_CONSTEXPR (SetOpposite) {
|
||||
kernel.assignOppositeCoeff(row, col);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -942,22 +945,19 @@ struct triangular_assignment_loop<Kernel, Mode, Dynamic, SetOpposite> {
|
||||
const Index maxi = numext::mini(outer, innerSize);
|
||||
Index i = 0;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(ActiveBeforeDiag) {
|
||||
EIGEN_IF_CONSTEXPR (ActiveBeforeDiag) {
|
||||
for (; i < maxi; ++i) kernel.assignCoeff(row(outer, i), col(outer, i));
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(SetOpposite) {
|
||||
} else EIGEN_IF_CONSTEXPR (SetOpposite) {
|
||||
for (; i < maxi; ++i) kernel.assignOppositeCoeff(row(outer, i), col(outer, i));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
i = maxi;
|
||||
}
|
||||
|
||||
if (i < innerSize) kernel.assignDiagonalCoeff(i++);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(!ActiveBeforeDiag) {
|
||||
EIGEN_IF_CONSTEXPR (!ActiveBeforeDiag) {
|
||||
for (; i < innerSize; ++i) kernel.assignCoeff(row(outer, i), col(outer, i));
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(SetOpposite) {
|
||||
} else EIGEN_IF_CONSTEXPR (SetOpposite) {
|
||||
for (; i < innerSize; ++i) kernel.assignOppositeCoeff(row(outer, i), col(outer, i));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1001,10 +1001,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
|
||||
Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
|
||||
Index count = 0;
|
||||
const Index peeled_k = (depth / PacketSize) * PacketSize;
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
// skip what we have before
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
|
||||
const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
|
||||
const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
|
||||
@@ -1014,7 +1014,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
|
||||
const LinearMapper dm6 = rhs.getLinearMapper(0, j2 + 6);
|
||||
const LinearMapper dm7 = rhs.getLinearMapper(0, j2 + 7);
|
||||
Index k = 0;
|
||||
EIGEN_IF_CONSTEXPR((PacketSize % 8) == 0 || PacketSize == 4) {
|
||||
EIGEN_IF_CONSTEXPR ((PacketSize % 8) == 0 || PacketSize == 4) {
|
||||
for (; k < peeled_k; k += PacketSize) {
|
||||
PacketBlock<Packet, 8> kernel;
|
||||
|
||||
@@ -1027,7 +1027,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
|
||||
kernel.packet[6] = dm6.template loadPacket<Packet>(k);
|
||||
kernel.packet[7] = dm7.template loadPacket<Packet>(k);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(PacketSize == 4) {
|
||||
EIGEN_IF_CONSTEXPR (PacketSize == 4) {
|
||||
// For PacketSize==4 we cannot ptranspose 8 packets directly; compose two
|
||||
// 4-packet transposes (cols 0-3 and 4-7) and interleave the halves so
|
||||
// the 8 stores produce 4 rows of 8 packed elements.
|
||||
@@ -1051,8 +1051,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
|
||||
kernel.packet[5] = tmp_hi.packet[2];
|
||||
kernel.packet[6] = tmp_lo.packet[3];
|
||||
kernel.packet[7] = tmp_hi.packet[3];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
ptranspose(kernel);
|
||||
}
|
||||
|
||||
@@ -1079,28 +1078,28 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
|
||||
count += 8;
|
||||
}
|
||||
// skip what we have after
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(nr >= 4) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 4) {
|
||||
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
|
||||
// skip what we have before
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
|
||||
const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
|
||||
const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
|
||||
const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3);
|
||||
|
||||
Index k = 0;
|
||||
EIGEN_IF_CONSTEXPR((PacketSize % 4) == 0 || PacketSize == 2) {
|
||||
EIGEN_IF_CONSTEXPR ((PacketSize % 4) == 0 || PacketSize == 2) {
|
||||
for (; k < peeled_k; k += PacketSize) {
|
||||
PacketBlock<Packet, 4> kernel;
|
||||
kernel.packet[0] = dm0.template loadPacket<Packet>(k);
|
||||
kernel.packet[1] = dm1.template loadPacket<Packet>(k);
|
||||
kernel.packet[2] = dm2.template loadPacket<Packet>(k);
|
||||
kernel.packet[3] = dm3.template loadPacket<Packet>(k);
|
||||
EIGEN_IF_CONSTEXPR(PacketSize == 2) {
|
||||
EIGEN_IF_CONSTEXPR (PacketSize == 2) {
|
||||
// See the matching note in GeneralBlockPanelKernel.h.
|
||||
PacketBlock<Packet, 2> tmp01;
|
||||
tmp01.packet[0] = kernel.packet[0];
|
||||
@@ -1114,8 +1113,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
|
||||
kernel.packet[1] = tmp23.packet[0];
|
||||
kernel.packet[2] = tmp01.packet[1];
|
||||
kernel.packet[3] = tmp23.packet[1];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
ptranspose(kernel);
|
||||
}
|
||||
pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0]));
|
||||
@@ -1133,19 +1131,19 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Con
|
||||
count += 4;
|
||||
}
|
||||
// skip what we have after
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
// copy the remaining columns one at a time (nr==1)
|
||||
for (Index j2 = packet_cols4; j2 < cols; ++j2) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(0, j2);
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
blockB[count] = cj(dm0(k));
|
||||
count += 1;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1174,33 +1172,29 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, 8, RowMajor, Conjugate, PanelMod
|
||||
Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
|
||||
Index count = 0;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
// skip what we have before
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
EIGEN_IF_CONSTEXPR(PacketSize == 8) {
|
||||
EIGEN_IF_CONSTEXPR (PacketSize == 8) {
|
||||
// Packet A = ploadu<Packet>(&rhs.data()[k*rhs.stride() + j2]);
|
||||
Packet A = rhs.template loadPacket<Packet>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 8) {
|
||||
} else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 8) {
|
||||
HalfPacket A = rhs.template loadPacket<HalfPacket>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 8) {
|
||||
} else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 8) {
|
||||
QuarterPacket A = rhs.template loadPacket<QuarterPacket>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(PacketSize == 4) {
|
||||
} else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
|
||||
// Packet A = ploadu<Packet>(&rhs.data()[k*rhs.stride() + j2]);
|
||||
// Packet B = ploadu<Packet>(&rhs.data()[k*rhs.stride() + j2 + PacketSize]);
|
||||
Packet A = rhs.template loadPacket<Packet>(k, j2);
|
||||
Packet B = rhs.template loadPacket<Packet>(k, j2 + PacketSize);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
pstoreu(blockB + count + PacketSize, cj.pconj(B));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// const Scalar* b0 = &rhs.data()[k*rhs.stride() + j2];
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
|
||||
blockB[count + 0] = cj(dm0(0));
|
||||
@@ -1215,31 +1209,28 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, 8, RowMajor, Conjugate, PanelMod
|
||||
count += 8;
|
||||
}
|
||||
// skip what we have after
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(nr >= 4) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 4) {
|
||||
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
|
||||
// skip what we have before
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
EIGEN_IF_CONSTEXPR(PacketSize == 4) {
|
||||
EIGEN_IF_CONSTEXPR (PacketSize == 4) {
|
||||
Packet A = rhs.template loadPacket<Packet>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
count += PacketSize;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 4) {
|
||||
} else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 4) {
|
||||
HalfPacket A = rhs.template loadPacket<HalfPacket>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
count += HalfPacketSize;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 4) {
|
||||
} else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 4) {
|
||||
QuarterPacket A = rhs.template loadPacket<QuarterPacket>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
count += QuarterPacketSize;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
|
||||
blockB[count + 0] = cj(dm0(0));
|
||||
blockB[count + 1] = cj(dm0(1));
|
||||
@@ -1249,17 +1240,17 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, 8, RowMajor, Conjugate, PanelMod
|
||||
}
|
||||
}
|
||||
// skip what we have after
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
// copy the remaining columns one at a time (nr==1)
|
||||
for (Index j2 = packet_cols4; j2 < cols; ++j2) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
blockB[count] = cj(rhs(k, j2));
|
||||
count += 1;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += stride - offset - depth;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += stride - offset - depth;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -136,23 +136,22 @@ EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_
|
||||
static_assert(unrollM == EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW");
|
||||
|
||||
urolls::template transpose<unrollN, 0>(zmm);
|
||||
EIGEN_IF_CONSTEXPR(unrollN > U2) urolls::template transpose<unrollN, 2>(zmm);
|
||||
EIGEN_IF_CONSTEXPR(unrollN > U1) urolls::template transpose<unrollN, 1>(zmm);
|
||||
EIGEN_IF_CONSTEXPR (unrollN > U2) urolls::template transpose<unrollN, 2>(zmm);
|
||||
EIGEN_IF_CONSTEXPR (unrollN > U1) urolls::template transpose<unrollN, 1>(zmm);
|
||||
|
||||
static_assert((remN && unrollN == U1) || !remN, "When handling N remainder set unrollN=U1");
|
||||
EIGEN_IF_CONSTEXPR(!remN) {
|
||||
EIGEN_IF_CONSTEXPR (!remN) {
|
||||
urolls::template storeC<std::min(unrollN, U1), unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
|
||||
EIGEN_IF_CONSTEXPR(unrollN > U1) {
|
||||
EIGEN_IF_CONSTEXPR (unrollN > U1) {
|
||||
constexpr int64_t unrollN_ = std::min(unrollN - U1, U1);
|
||||
urolls::template storeC<unrollN_, unrollN, 1, remM>(C_arr + U1 * LDC, LDC, zmm, remM_);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(unrollN > U2) {
|
||||
EIGEN_IF_CONSTEXPR (unrollN > U2) {
|
||||
constexpr int64_t unrollN_ = std::min(unrollN - U2, U1);
|
||||
urolls::template storeC<unrollN_, unrollN, 2, remM>(C_arr + U2 * LDC, LDC, zmm, remM_);
|
||||
}
|
||||
}
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR((std::is_same<Scalar, float>::value)) {
|
||||
} else {
|
||||
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, float>::value)) {
|
||||
// Note: without "if constexpr" this section of code will also be
|
||||
// parsed by the compiler so each of the storeC will still be instantiated.
|
||||
// We use enable_if in aux_storeC to set it to an empty function for
|
||||
@@ -187,8 +186,7 @@ EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_
|
||||
urolls::template storeC<2, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
|
||||
else if (remN_ == 1)
|
||||
urolls::template storeC<1, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
if (remN_ == 7)
|
||||
urolls::template storeC<7, unrollN, 0, remM>(C_arr, LDC, zmm, remM_);
|
||||
else if (remN_ == 6)
|
||||
@@ -244,23 +242,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 3, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 3, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_B_LOAD_SETS * 3,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, false, false>(zmm, &C_arr[i + j * LDC], LDC);
|
||||
}
|
||||
}
|
||||
@@ -273,23 +274,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 3, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_B_LOAD_SETS * 3,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 3, 4, 1, EIGEN_AVX_B_LOAD_SETS * 3, EIGEN_AVX_MAX_A_BCAST>(
|
||||
B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<3, 4>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<3, 4>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, true, false>(zmm, &C_arr[i + j * LDC], LDC, 4);
|
||||
}
|
||||
i += 4;
|
||||
@@ -303,23 +307,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 3, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_B_LOAD_SETS * 3,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 3, 2, 1, EIGEN_AVX_B_LOAD_SETS * 3, EIGEN_AVX_MAX_A_BCAST>(
|
||||
B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<3, 2>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<3, 2>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, true, false>(zmm, &C_arr[i + j * LDC], LDC, 2);
|
||||
}
|
||||
i += 2;
|
||||
@@ -334,22 +341,25 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 3, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_B_LOAD_SETS * 3, 1>(
|
||||
B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 3, 1, 1, EIGEN_AVX_B_LOAD_SETS * 3, 1>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<3, 1>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<3, 1>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U3, true, false>(zmm, &C_arr[i + j * LDC], LDC, 1);
|
||||
}
|
||||
}
|
||||
@@ -360,30 +370,33 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
int64_t i = 0;
|
||||
for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) {
|
||||
Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j];
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) B_t = &B_arr[0 * LDB + j];
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) B_t = &B_arr[0 * LDB + j];
|
||||
PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm;
|
||||
urolls::template setzero<2, EIGEN_AVX_MAX_NUM_ROW>(zmm);
|
||||
for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) {
|
||||
urolls::template microKernel<isARowMajor, 2, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 2, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, false, false>(zmm, &C_arr[i + j * LDC], LDC);
|
||||
}
|
||||
}
|
||||
@@ -396,23 +409,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 2, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 2, 4, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
|
||||
LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<2, 4>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<2, 4>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, true, false>(zmm, &C_arr[i + j * LDC], LDC, 4);
|
||||
}
|
||||
i += 4;
|
||||
@@ -426,23 +442,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 2, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 2, 2, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
|
||||
LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<2, 2>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<2, 2>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, true, false>(zmm, &C_arr[i + j * LDC], LDC, 2);
|
||||
}
|
||||
i += 2;
|
||||
@@ -456,22 +475,25 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 2, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD, 1>(B_t, A_t, LDB,
|
||||
LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 2, 1, 1, EIGEN_AVX_MAX_B_LOAD, 1>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<2, 1>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<2, 1>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U2, true, false>(zmm, &C_arr[i + j * LDC], LDC, 1);
|
||||
}
|
||||
}
|
||||
@@ -488,23 +510,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_B_LOAD_SETS * 1,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, false, false>(zmm, &C_arr[i + j * LDC], LDC);
|
||||
}
|
||||
}
|
||||
@@ -517,23 +542,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 1, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 1, 4, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
|
||||
LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<1, 4>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<1, 4>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, false>(zmm, &C_arr[i + j * LDC], LDC, 4);
|
||||
}
|
||||
i += 4;
|
||||
@@ -547,23 +575,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 1, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 1, 2, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST>(B_t, A_t, LDB,
|
||||
LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<1, 2>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<1, 2>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, false>(zmm, &C_arr[i + j * LDC], LDC, 2);
|
||||
}
|
||||
i += 2;
|
||||
@@ -578,22 +609,25 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 1, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD, 1>(B_t, A_t, LDB,
|
||||
LDA, zmm);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 1, 1, 1, EIGEN_AVX_B_LOAD_SETS * 1, 1>(B_t, A_t, LDB, LDA, zmm);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<1, 1>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
urolls::template storeC<1, 1>(&C_arr[i * LDC + j], LDC, zmm);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, false>(zmm, &C_arr[i + j * LDC], LDC, 1);
|
||||
}
|
||||
}
|
||||
@@ -612,23 +646,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 1, EIGEN_AVX_MAX_NUM_ROW, 1, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
|
||||
urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, false, true>(zmm, &C_arr[i + j * LDC], LDC, 0, N - j);
|
||||
}
|
||||
}
|
||||
@@ -641,23 +678,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 1, 4, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 1, 4, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST, true>(
|
||||
B_t, A_t, LDB, LDA, zmm, N - j);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
|
||||
urolls::template storeC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, true>(zmm, &C_arr[i + j * LDC], LDC, 4, N - j);
|
||||
}
|
||||
i += 4;
|
||||
@@ -671,23 +711,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 1, 2, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD,
|
||||
EIGEN_AVX_MAX_A_BCAST, true>(B_t, A_t, LDB, LDA, zmm, N - j);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 1, 2, 1, EIGEN_AVX_MAX_B_LOAD, EIGEN_AVX_MAX_A_BCAST, true>(
|
||||
B_t, A_t, LDB, LDA, zmm, N - j);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
|
||||
urolls::template storeC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, true>(zmm, &C_arr[i + j * LDC], LDC, 2, N - j);
|
||||
}
|
||||
i += 2;
|
||||
@@ -701,23 +744,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t
|
||||
urolls::template microKernel<isARowMajor, 1, 1, EIGEN_AVX_MAX_K_UNROL, EIGEN_AVX_MAX_B_LOAD, 1, true>(
|
||||
B_t, A_t, LDB, LDA, zmm, N - j);
|
||||
B_t += EIGEN_AVX_MAX_K_UNROL * LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL;
|
||||
else
|
||||
A_t += EIGEN_AVX_MAX_K_UNROL * LDA;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(handleKRem) {
|
||||
EIGEN_IF_CONSTEXPR (handleKRem) {
|
||||
for (int64_t k = K_; k < K; k++) {
|
||||
urolls::template microKernel<isARowMajor, 1, 1, 1, EIGEN_AVX_MAX_B_LOAD, 1, true>(B_t, A_t, LDB, LDA, zmm,
|
||||
N - j);
|
||||
B_t += LDB;
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) A_t++;
|
||||
else A_t += LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
A_t++;
|
||||
else
|
||||
A_t += LDA;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(isCRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isCRowMajor) {
|
||||
urolls::template updateC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
|
||||
urolls::template storeC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transStoreC<Scalar, vec, EIGEN_AVX_MAX_NUM_ROW, U1, true, true>(zmm, &C_arr[i + j * LDC], LDC, 1, N - j);
|
||||
}
|
||||
}
|
||||
@@ -840,7 +886,7 @@ EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar* B_arr, int64_t LDB, int64_t K,
|
||||
B_temp += U1;
|
||||
k += U1;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(U1 > 8) {
|
||||
EIGEN_IF_CONSTEXPR (U1 > 8) {
|
||||
// Note: without "if constexpr" this section of code will also be
|
||||
// parsed by the compiler so there is an additional check in {load/store}BBlock
|
||||
// to make sure the counter is not non-negative.
|
||||
@@ -850,7 +896,7 @@ EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar* B_arr, int64_t LDB, int64_t K,
|
||||
k += 8;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(U1 > 4) {
|
||||
EIGEN_IF_CONSTEXPR (U1 > 4) {
|
||||
// Note: without "if constexpr" this section of code will also be
|
||||
// parsed by the compiler so there is an additional check in {load/store}BBlock
|
||||
// to make sure the counter is not non-negative.
|
||||
@@ -921,7 +967,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
|
||||
int64_t sizeBTemp = 0;
|
||||
Scalar* B_temp = NULL;
|
||||
EIGEN_IF_CONSTEXPR(!isBRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (!isBRowMajor) {
|
||||
/**
|
||||
* If B is col-major, we copy it to a fixed-size temporary array of size at most ~numM*kB and
|
||||
* transpose it to row-major. Call the solve routine, and copy+transpose it back to the original array.
|
||||
@@ -930,7 +976,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
sizeBTemp = (((std::min(kB, numRHS) + psize - 1) / psize + 4) * psize) * numM;
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar*)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64);
|
||||
EIGEN_IF_CONSTEXPR (!isBRowMajor) B_temp = (Scalar*)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64);
|
||||
|
||||
for (int64_t k = 0; k < numRHS; k += kB) {
|
||||
int64_t bK = numRHS - k > kB ? kB : numRHS - k;
|
||||
@@ -944,7 +990,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
int64_t LDT = ((bkL + (numScalarPerCache - 1)) / numScalarPerCache) * numScalarPerCache;
|
||||
int64_t offsetBTemp = 0;
|
||||
for (int64_t i = 0; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) {
|
||||
EIGEN_IF_CONSTEXPR(!isBRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (!isBRowMajor) {
|
||||
int64_t indA_i = isFWDSolve ? i : M - 1 - i;
|
||||
int64_t indB_i = isFWDSolve ? i : M - (i + EIGEN_AVX_MAX_NUM_ROW);
|
||||
int64_t offB_1 = isFWDSolve ? offsetBTemp : sizeBTemp - EIGEN_AVX_MAX_NUM_ROW * LDT - offsetBTemp;
|
||||
@@ -958,8 +1004,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
copyBToRowMajor<Scalar, false, false>(B_arr + indB_i + k * LDB, LDB, bK, B_temp + offB_1, LDT);
|
||||
|
||||
offsetBTemp += EIGEN_AVX_MAX_NUM_ROW * LDT;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
int64_t ind = isFWDSolve ? i : M - 1 - i;
|
||||
triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(
|
||||
&A_arr[idA<isARowMajor>(ind, ind, LDA)], B_arr + k + ind * LDB, EIGEN_AVX_MAX_NUM_ROW, bK, LDA, LDB);
|
||||
@@ -977,7 +1022,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
* |__|__|__|__ |__|
|
||||
* |********|__| |**|
|
||||
*/
|
||||
EIGEN_IF_CONSTEXPR(isBRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isBRowMajor) {
|
||||
int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW);
|
||||
int64_t indA_j = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW);
|
||||
int64_t indB_i = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW);
|
||||
@@ -985,8 +1030,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
gemmKernel<Scalar, isARowMajor, isBRowMajor, false, false>(
|
||||
&A_arr[idA<isARowMajor>(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB,
|
||||
EIGEN_AVX_MAX_NUM_ROW, bK, i + EIGEN_AVX_MAX_NUM_ROW, LDA, LDB, LDB);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
if (offsetBTemp + EIGEN_AVX_MAX_NUM_ROW * LDT > sizeBTemp) {
|
||||
/**
|
||||
* Similar idea as mentioned above, but here we are limited by the number of updated values of B
|
||||
@@ -1024,7 +1068,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
int64_t bM = M - M_;
|
||||
if (bM > 0) {
|
||||
if (M_ > 0) {
|
||||
EIGEN_IF_CONSTEXPR(isBRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (isBRowMajor) {
|
||||
int64_t indA_i = isFWDSolve ? M_ : 0;
|
||||
int64_t indA_j = isFWDSolve ? 0 : bM;
|
||||
int64_t indB_i = isFWDSolve ? 0 : bM;
|
||||
@@ -1032,8 +1076,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
gemmKernel<Scalar, isARowMajor, isBRowMajor, false, false>(
|
||||
&A_arr[idA<isARowMajor>(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB, bM,
|
||||
bK, M_, LDA, LDB, LDB);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
int64_t indA_i = isFWDSolve ? M_ : 0;
|
||||
int64_t indA_j = isFWDSolve ? gemmOff : bM;
|
||||
int64_t indB_i = isFWDSolve ? M_ : 0;
|
||||
@@ -1043,7 +1086,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
M_ - gemmOff, LDA, LDT, LDB);
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(!isBRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (!isBRowMajor) {
|
||||
int64_t indA_i = isFWDSolve ? M_ : M - 1 - M_;
|
||||
int64_t indB_i = isFWDSolve ? M_ : 0;
|
||||
int64_t offB_1 = isFWDSolve ? 0 : (bM - 1) * bkL;
|
||||
@@ -1051,8 +1094,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(&A_arr[idA<isARowMajor>(indA_i, indA_i, LDA)],
|
||||
B_temp + offB_1, bM, bkL, LDA, bkL);
|
||||
copyBToRowMajor<Scalar, false, true>(B_arr + indB_i + k * LDB, LDB, bK, B_temp, bkL, bM);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
int64_t ind = isFWDSolve ? M_ : M - 1 - M_;
|
||||
triSolveKernelLxK<Scalar, isARowMajor, isFWDSolve, isUnitDiag>(&A_arr[idA<isARowMajor>(ind, ind, LDA)],
|
||||
B_arr + k + ind * LDB, bM, bK, LDA, LDB);
|
||||
@@ -1060,7 +1102,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(!isBRowMajor) handmade_aligned_free(B_temp);
|
||||
EIGEN_IF_CONSTEXPR (!isBRowMajor) handmade_aligned_free(B_temp);
|
||||
}
|
||||
|
||||
// Template specializations of trsmKernelL/R for float/double and inner strides of 1.
|
||||
|
||||
@@ -13,8 +13,10 @@
|
||||
|
||||
template <bool isARowMajor = true>
|
||||
EIGEN_ALWAYS_INLINE int64_t idA(int64_t i, int64_t j, int64_t LDA) {
|
||||
EIGEN_IF_CONSTEXPR(isARowMajor) return i * LDA + j;
|
||||
else return i + j * LDA;
|
||||
EIGEN_IF_CONSTEXPR (isARowMajor)
|
||||
return i * LDA + j;
|
||||
else
|
||||
return i + j * LDA;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -59,11 +61,11 @@ namespace unrolls {
|
||||
|
||||
template <int64_t N>
|
||||
EIGEN_ALWAYS_INLINE auto remMask(int64_t m) {
|
||||
EIGEN_IF_CONSTEXPR(N == 16) { return 0xFFFF >> (16 - m); }
|
||||
else EIGEN_IF_CONSTEXPR(N == 8) {
|
||||
EIGEN_IF_CONSTEXPR (N == 16) {
|
||||
return 0xFFFF >> (16 - m);
|
||||
} else EIGEN_IF_CONSTEXPR (N == 8) {
|
||||
return 0xFF >> (8 - m);
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(N == 4) {
|
||||
} else EIGEN_IF_CONSTEXPR (N == 4) {
|
||||
return 0x0F >> (4 - m);
|
||||
}
|
||||
return 0;
|
||||
@@ -155,22 +157,20 @@ class trans {
|
||||
constexpr int64_t counterReverse = endN - counter;
|
||||
constexpr int64_t startN = counterReverse;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(startN < EIGEN_AVX_MAX_NUM_ROW) {
|
||||
EIGEN_IF_CONSTEXPR(remM) {
|
||||
EIGEN_IF_CONSTEXPR (startN < EIGEN_AVX_MAX_NUM_ROW) {
|
||||
EIGEN_IF_CONSTEXPR (remM) {
|
||||
pstoreu<Scalar>(
|
||||
C_arr + LDC * startN,
|
||||
padd(ploadu<vecHalf>((const Scalar *)C_arr + LDC * startN, remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_)),
|
||||
preinterpret<vecHalf>(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN]),
|
||||
remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_)),
|
||||
remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
pstoreu<Scalar>(C_arr + LDC * startN,
|
||||
padd(ploadu<vecHalf>((const Scalar *)C_arr + LDC * startN),
|
||||
preinterpret<vecHalf>(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN])));
|
||||
}
|
||||
}
|
||||
else { // This block is only needed for fp32 case
|
||||
} else { // This block is only needed for fp32 case
|
||||
// Reinterpret as __m512 for _mm512_shuffle_f32x4
|
||||
vecFullFloat zmm2vecFullFloat = preinterpret<vecFullFloat>(
|
||||
zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)]);
|
||||
@@ -178,15 +178,14 @@ class trans {
|
||||
zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)] =
|
||||
preinterpret<vec>(_mm512_shuffle_f32x4(zmm2vecFullFloat, zmm2vecFullFloat, 0b01001110));
|
||||
|
||||
EIGEN_IF_CONSTEXPR(remM) {
|
||||
EIGEN_IF_CONSTEXPR (remM) {
|
||||
pstoreu<Scalar>(
|
||||
C_arr + LDC * startN,
|
||||
padd(ploadu<vecHalf>((const Scalar *)C_arr + LDC * startN, remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_)),
|
||||
preinterpret<vecHalf>(
|
||||
zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)])),
|
||||
remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
pstoreu<Scalar>(
|
||||
C_arr + LDC * startN,
|
||||
padd(ploadu<vecHalf>((const Scalar *)C_arr + LDC * startN),
|
||||
@@ -308,16 +307,15 @@ class transB {
|
||||
constexpr int64_t counterReverse = endN - counter;
|
||||
constexpr int64_t startN = counterReverse;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(remM) {
|
||||
EIGEN_IF_CONSTEXPR (remM) {
|
||||
ymm.packet[packetIndexOffset + startN] =
|
||||
ploadu<vecHalf>((const Scalar *)&B_arr[startN * LDB], remMask<EIGEN_AVX_MAX_NUM_ROW>(remM_));
|
||||
}
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR(remN_ == 0) {
|
||||
} else {
|
||||
EIGEN_IF_CONSTEXPR (remN_ == 0) {
|
||||
ymm.packet[packetIndexOffset + startN] = ploadu<vecHalf>((const Scalar *)&B_arr[startN * LDB]);
|
||||
}
|
||||
else ymm.packet[packetIndexOffset + startN] =
|
||||
ploadu<vecHalf>((const Scalar *)&B_arr[startN * LDB], remMask<EIGEN_AVX_MAX_NUM_ROW>(remN_));
|
||||
} else
|
||||
ymm.packet[packetIndexOffset + startN] =
|
||||
ploadu<vecHalf>((const Scalar *)&B_arr[startN * LDB], remMask<EIGEN_AVX_MAX_NUM_ROW>(remN_));
|
||||
}
|
||||
|
||||
aux_loadB<endN, counter - 1, packetIndexOffset, remM, remN_>(B_arr, LDB, ymm, remM_);
|
||||
@@ -345,11 +343,10 @@ class transB {
|
||||
constexpr int64_t counterReverse = endN - counter;
|
||||
constexpr int64_t startN = counterReverse;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(remK || remM) {
|
||||
EIGEN_IF_CONSTEXPR (remK || remM) {
|
||||
pstoreu<Scalar>(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN],
|
||||
remMask<EIGEN_AVX_MAX_NUM_ROW>(rem_));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
pstoreu<Scalar>(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN]);
|
||||
}
|
||||
|
||||
@@ -406,10 +403,9 @@ class transB {
|
||||
constexpr int64_t counterReverse = endN - counter;
|
||||
constexpr int64_t startN = counterReverse;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(toTemp) {
|
||||
EIGEN_IF_CONSTEXPR (toTemp) {
|
||||
transB::template storeB<EIGEN_AVX_MAX_NUM_ROW, startN, remK_ != 0, false>(&B_temp[startN], LDB_, ymm, remK_);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
transB::template storeB<std::min(EIGEN_AVX_MAX_NUM_ROW, endN), startN, false, remM>(&B_arr[0 + startN * LDB], LDB,
|
||||
ymm, remM_);
|
||||
}
|
||||
@@ -450,8 +446,9 @@ class transB {
|
||||
static EIGEN_ALWAYS_INLINE void loadBBlock(Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_,
|
||||
PacketBlock<vecHalf, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> &ymm,
|
||||
int64_t remM_ = 0) {
|
||||
EIGEN_IF_CONSTEXPR(toTemp) { transB::template loadB<unrollN, 0, remM, 0>(&B_arr[0], LDB, ymm, remM_); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (toTemp) {
|
||||
transB::template loadB<unrollN, 0, remM, 0>(&B_arr[0], LDB, ymm, remM_);
|
||||
} else {
|
||||
aux_loadBBlock<unrollN, unrollN, toTemp, remM, remN_>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
}
|
||||
}
|
||||
@@ -501,7 +498,7 @@ class transB {
|
||||
*
|
||||
* For fp32 L and U1 are 1:2 so for U3/U2 cases the loads/stores need to be split up.
|
||||
*/
|
||||
EIGEN_IF_CONSTEXPR(unrollN == U3) {
|
||||
EIGEN_IF_CONSTEXPR (unrollN == U3) {
|
||||
// load LxU3 B col major, transpose LxU3 row major
|
||||
constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U3);
|
||||
transB::template loadBBlock<maxUBlock, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
@@ -510,7 +507,7 @@ class transB {
|
||||
transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
|
||||
transB::template storeBBlock<maxUBlock, toTemp, remM, 0>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(maxUBlock < U3) {
|
||||
EIGEN_IF_CONSTEXPR (maxUBlock < U3) {
|
||||
transB::template loadBBlock<maxUBlock, toTemp, remM>(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_,
|
||||
ymm, remM_);
|
||||
transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
|
||||
@@ -519,50 +516,46 @@ class transB {
|
||||
transB::template storeBBlock<maxUBlock, toTemp, remM, 0>(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_,
|
||||
ymm, remM_);
|
||||
}
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(unrollN == U2) {
|
||||
} else EIGEN_IF_CONSTEXPR (unrollN == U2) {
|
||||
// load LxU2 B col major, transpose LxU2 row major
|
||||
constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U2);
|
||||
transB::template loadBBlock<maxUBlock, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
|
||||
transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
|
||||
EIGEN_IF_CONSTEXPR(maxUBlock < U2) transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
|
||||
EIGEN_IF_CONSTEXPR (maxUBlock < U2) transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
|
||||
transB::template storeBBlock<maxUBlock, toTemp, remM, 0>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(maxUBlock < U2) {
|
||||
EIGEN_IF_CONSTEXPR (maxUBlock < U2) {
|
||||
transB::template loadBBlock<EIGEN_AVX_MAX_NUM_ROW, toTemp, remM>(&B_arr[maxUBlock * LDB], LDB,
|
||||
&B_temp[maxUBlock], LDB_, ymm, remM_);
|
||||
transB::template transposeLxL<0>(ymm);
|
||||
transB::template storeBBlock<EIGEN_AVX_MAX_NUM_ROW, toTemp, remM, 0>(&B_arr[maxUBlock * LDB], LDB,
|
||||
&B_temp[maxUBlock], LDB_, ymm, remM_);
|
||||
}
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(unrollN == U1) {
|
||||
} else EIGEN_IF_CONSTEXPR (unrollN == U1) {
|
||||
// load LxU1 B col major, transpose LxU1 row major
|
||||
transB::template loadBBlock<U1, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
transB::template transposeLxL<0>(ymm);
|
||||
EIGEN_IF_CONSTEXPR(EIGEN_AVX_MAX_NUM_ROW < U1) { transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); }
|
||||
EIGEN_IF_CONSTEXPR (EIGEN_AVX_MAX_NUM_ROW < U1) {
|
||||
transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm);
|
||||
}
|
||||
transB::template storeBBlock<U1, toTemp, remM, 0>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(unrollN == 8 && U1 > 8) {
|
||||
} else EIGEN_IF_CONSTEXPR (unrollN == 8 && U1 > 8) {
|
||||
// load Lx4 B col major, transpose Lx4 row major
|
||||
transB::template loadBBlock<8, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
transB::template transposeLxL<0>(ymm);
|
||||
transB::template storeBBlock<8, toTemp, remM, 8>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(unrollN == 4 && U1 > 4) {
|
||||
} else EIGEN_IF_CONSTEXPR (unrollN == 4 && U1 > 4) {
|
||||
// load Lx4 B col major, transpose Lx4 row major
|
||||
transB::template loadBBlock<4, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
transB::template transposeLxL<0>(ymm);
|
||||
transB::template storeBBlock<4, toTemp, remM, 4>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(unrollN == 2) {
|
||||
} else EIGEN_IF_CONSTEXPR (unrollN == 2) {
|
||||
// load Lx2 B col major, transpose Lx2 row major
|
||||
transB::template loadBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
transB::template transposeLxL<0>(ymm);
|
||||
transB::template storeBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(unrollN == 1) {
|
||||
} else EIGEN_IF_CONSTEXPR (unrollN == 1) {
|
||||
// load Lx1 B col major, transpose Lx1 row major
|
||||
transB::template loadBBlock<1, toTemp, remM, 1>(B_arr, LDB, B_temp, LDB_, ymm, remM_);
|
||||
transB::template transposeLxL<0>(ymm);
|
||||
@@ -614,10 +607,9 @@ class trsm {
|
||||
constexpr int64_t packetIndex = startM * endK + startK;
|
||||
constexpr int64_t startM_ = isFWDSolve ? startM : -startM;
|
||||
const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB;
|
||||
EIGEN_IF_CONSTEXPR(krem) {
|
||||
EIGEN_IF_CONSTEXPR (krem) {
|
||||
RHSInPacket.packet[packetIndex] = ploadu<vec>(&B_arr[rhsIndex], remMask<PacketSize>(rem));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
RHSInPacket.packet[packetIndex] = ploadu<vec>(&B_arr[rhsIndex]);
|
||||
}
|
||||
aux_loadRHS<isFWDSolve, endM, endK, counter - 1, krem>(B_arr, LDB, RHSInPacket, rem);
|
||||
@@ -649,10 +641,9 @@ class trsm {
|
||||
constexpr int64_t packetIndex = startM * endK + startK;
|
||||
constexpr int64_t startM_ = isFWDSolve ? startM : -startM;
|
||||
const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB;
|
||||
EIGEN_IF_CONSTEXPR(krem) {
|
||||
EIGEN_IF_CONSTEXPR (krem) {
|
||||
pstoreu<Scalar>(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex], remMask<PacketSize>(rem));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
pstoreu<Scalar>(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex]);
|
||||
}
|
||||
aux_storeRHS<isFWDSolve, endM, endK, counter - 1, krem>(B_arr, LDB, RHSInPacket, rem);
|
||||
@@ -711,26 +702,27 @@ class trsm {
|
||||
|
||||
// For each row of A, first update all corresponding RHS
|
||||
constexpr int64_t packetIndex = startM * endK + startK;
|
||||
EIGEN_IF_CONSTEXPR(currentM > 0) {
|
||||
EIGEN_IF_CONSTEXPR (currentM > 0) {
|
||||
RHSInPacket.packet[packetIndex] =
|
||||
pnmadd(AInPacket.packet[startM], RHSInPacket.packet[(currentM - 1) * endK + startK],
|
||||
RHSInPacket.packet[packetIndex]);
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(startK == endK - 1) {
|
||||
EIGEN_IF_CONSTEXPR (startK == endK - 1) {
|
||||
// Once all RHS for previous row of A is updated, we broadcast the next element in the column A_{i, currentM}.
|
||||
EIGEN_IF_CONSTEXPR(startM == currentM && !isUnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR (startM == currentM && !isUnitDiag) {
|
||||
// If diagonal is not unit, we broadcast reciprocals of diagonals AinPacket.packet[currentM].
|
||||
// This will be used in divRHSByDiag
|
||||
EIGEN_IF_CONSTEXPR(isFWDSolve)
|
||||
AInPacket.packet[currentM] = pset1<vec>(Scalar(1) / A_arr[idA<isARowMajor>(currentM, currentM, LDA)]);
|
||||
else AInPacket.packet[currentM] = pset1<vec>(Scalar(1) / A_arr[idA<isARowMajor>(-currentM, -currentM, LDA)]);
|
||||
}
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (isFWDSolve)
|
||||
AInPacket.packet[currentM] = pset1<vec>(Scalar(1) / A_arr[idA<isARowMajor>(currentM, currentM, LDA)]);
|
||||
else
|
||||
AInPacket.packet[currentM] = pset1<vec>(Scalar(1) / A_arr[idA<isARowMajor>(-currentM, -currentM, LDA)]);
|
||||
} else {
|
||||
// Broadcast next off diagonal element of A
|
||||
EIGEN_IF_CONSTEXPR(isFWDSolve)
|
||||
AInPacket.packet[startM] = pset1<vec>(A_arr[idA<isARowMajor>(startM, currentM, LDA)]);
|
||||
else AInPacket.packet[startM] = pset1<vec>(A_arr[idA<isARowMajor>(-startM, -currentM, LDA)]);
|
||||
EIGEN_IF_CONSTEXPR (isFWDSolve)
|
||||
AInPacket.packet[startM] = pset1<vec>(A_arr[idA<isARowMajor>(startM, currentM, LDA)]);
|
||||
else
|
||||
AInPacket.packet[startM] = pset1<vec>(A_arr[idA<isARowMajor>(-startM, -currentM, LDA)]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -769,8 +761,8 @@ class trsm {
|
||||
// Without "if constexpr" the compiler instantiates the case <-1, numK>
|
||||
// this is handled with enable_if to prevent out-of-bound warnings
|
||||
// from the compiler
|
||||
EIGEN_IF_CONSTEXPR(!isUnitDiag && startM > 0)
|
||||
trsm::template divRHSByDiag<startM - 1, numK>(RHSInPacket, AInPacket);
|
||||
EIGEN_IF_CONSTEXPR (!isUnitDiag && startM > 0)
|
||||
trsm::template divRHSByDiag<startM - 1, numK>(RHSInPacket, AInPacket);
|
||||
|
||||
// After division, the rhs corresponding to subsequent rows of A can be partially updated
|
||||
// We also broadcast the reciprocal of the next diagonal to AInPacket.packet[currentM] (if needed)
|
||||
@@ -779,8 +771,8 @@ class trsm {
|
||||
AInPacket);
|
||||
|
||||
// Handle division for the RHS corresponding to the final row of A.
|
||||
EIGEN_IF_CONSTEXPR(!isUnitDiag && startM == endM - 1)
|
||||
trsm::template divRHSByDiag<startM, numK>(RHSInPacket, AInPacket);
|
||||
EIGEN_IF_CONSTEXPR (!isUnitDiag && startM == endM - 1)
|
||||
trsm::template divRHSByDiag<startM, numK>(RHSInPacket, AInPacket);
|
||||
|
||||
aux_triSolveMicroKernel<isARowMajor, isFWDSolve, isUnitDiag, endM, counter - 1, numK>(A_arr, LDA, RHSInPacket,
|
||||
AInPacket);
|
||||
@@ -916,12 +908,13 @@ class gemm {
|
||||
constexpr int64_t startM = counterReverse / (endN);
|
||||
constexpr int64_t startN = counterReverse % endN;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(rem)
|
||||
zmm.packet[startN * endM + startM] =
|
||||
padd(ploadu<vec>(&C_arr[(startN)*LDC + startM * PacketSize], remMask<PacketSize>(rem_)),
|
||||
zmm.packet[startN * endM + startM], remMask<PacketSize>(rem_));
|
||||
else zmm.packet[startN * endM + startM] =
|
||||
padd(ploadu<vec>(&C_arr[(startN)*LDC + startM * PacketSize]), zmm.packet[startN * endM + startM]);
|
||||
EIGEN_IF_CONSTEXPR (rem)
|
||||
zmm.packet[startN * endM + startM] =
|
||||
padd(ploadu<vec>(&C_arr[(startN)*LDC + startM * PacketSize], remMask<PacketSize>(rem_)),
|
||||
zmm.packet[startN * endM + startM], remMask<PacketSize>(rem_));
|
||||
else
|
||||
zmm.packet[startN * endM + startM] =
|
||||
padd(ploadu<vec>(&C_arr[(startN)*LDC + startM * PacketSize]), zmm.packet[startN * endM + startM]);
|
||||
aux_updateC<endM, endN, counter - 1, rem>(C_arr, LDC, zmm, rem_);
|
||||
}
|
||||
|
||||
@@ -949,10 +942,11 @@ class gemm {
|
||||
constexpr int64_t startM = counterReverse / (endN);
|
||||
constexpr int64_t startN = counterReverse % endN;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(rem)
|
||||
pstoreu<Scalar>(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM],
|
||||
remMask<PacketSize>(rem_));
|
||||
else pstoreu<Scalar>(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM]);
|
||||
EIGEN_IF_CONSTEXPR (rem)
|
||||
pstoreu<Scalar>(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM],
|
||||
remMask<PacketSize>(rem_));
|
||||
else
|
||||
pstoreu<Scalar>(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM]);
|
||||
aux_storeC<endM, endN, counter - 1, rem>(C_arr, LDC, zmm, rem_);
|
||||
}
|
||||
|
||||
@@ -978,11 +972,12 @@ class gemm {
|
||||
constexpr int64_t counterReverse = endL - counter;
|
||||
constexpr int64_t startL = counterReverse;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(rem)
|
||||
zmm.packet[unrollM * unrollN + startL] =
|
||||
ploadu<vec>(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize], remMask<PacketSize>(rem_));
|
||||
else zmm.packet[unrollM * unrollN + startL] =
|
||||
ploadu<vec>(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize]);
|
||||
EIGEN_IF_CONSTEXPR (rem)
|
||||
zmm.packet[unrollM * unrollN + startL] =
|
||||
ploadu<vec>(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize], remMask<PacketSize>(rem_));
|
||||
else
|
||||
zmm.packet[unrollM * unrollN + startL] =
|
||||
ploadu<vec>(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize]);
|
||||
|
||||
aux_startLoadB<unrollM, unrollN, endL, counter - 1, rem>(B_t, LDB, zmm, rem_);
|
||||
}
|
||||
@@ -1037,11 +1032,10 @@ class gemm {
|
||||
constexpr int64_t counterReverse = endM - counter;
|
||||
constexpr int64_t startM = counterReverse;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(rem) {
|
||||
EIGEN_IF_CONSTEXPR (rem) {
|
||||
zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] =
|
||||
ploadu<vec>(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize], remMask<PacketSize>(rem_));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] =
|
||||
ploadu<vec>(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize]);
|
||||
}
|
||||
@@ -1079,32 +1073,31 @@ class gemm {
|
||||
constexpr int startN = (counterReverse / (endM)) % endN;
|
||||
constexpr int startM = counterReverse % endM;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(startK == 0 && startM == 0 && startN == 0) {
|
||||
EIGEN_IF_CONSTEXPR (startK == 0 && startM == 0 && startN == 0) {
|
||||
gemm::template startLoadB<endM, endN, numLoad, rem>(B_t, LDB, zmm, rem_);
|
||||
gemm::template startBCastA<isARowMajor, endM, endN, numBCast, numLoad>(A_t, LDA, zmm);
|
||||
}
|
||||
|
||||
{
|
||||
// Interleave FMA and Bcast
|
||||
EIGEN_IF_CONSTEXPR(isAdd) {
|
||||
EIGEN_IF_CONSTEXPR (isAdd) {
|
||||
zmm.packet[startN * endM + startM] =
|
||||
pmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast],
|
||||
zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
zmm.packet[startN * endM + startM] =
|
||||
pnmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast],
|
||||
zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]);
|
||||
}
|
||||
// Bcast
|
||||
EIGEN_IF_CONSTEXPR(startM == endM - 1 && (numBCast + startN + startK * endN < endK * endN)) {
|
||||
EIGEN_IF_CONSTEXPR (startM == endM - 1 && (numBCast + startN + startK * endN < endK * endN)) {
|
||||
zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast] = pload1<vec>(&A_t[idA<isARowMajor>(
|
||||
(numBCast + startN + startK * endN) % endN, (numBCast + startN + startK * endN) / endN, LDA)]);
|
||||
}
|
||||
}
|
||||
|
||||
// We have updated all accumulators, time to load next set of B's
|
||||
EIGEN_IF_CONSTEXPR((startN == endN - 1) && (startM == endM - 1)) {
|
||||
EIGEN_IF_CONSTEXPR ((startN == endN - 1) && (startM == endM - 1)) {
|
||||
gemm::template loadB<endM, endN, startK, endK, numLoad, numBCast, rem>(B_t, LDB, zmm, rem_);
|
||||
}
|
||||
aux_microKernel<isARowMajor, endM, endN, endK, counter - 1, numLoad, numBCast, rem>(B_t, A_t, LDB, LDA, zmm, rem_);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -59,11 +59,10 @@ EIGEN_STRONG_INLINE HVX_Vector HVX_load_partial(const T* mem) {
|
||||
HVX_Vector v0 = HVX_vmem<0>(mem);
|
||||
HVX_Vector v1 = v0;
|
||||
uintptr_t mem_addr = reinterpret_cast<uintptr_t>(mem);
|
||||
EIGEN_IF_CONSTEXPR(Size * sizeof(T) <= Alignment) {
|
||||
EIGEN_IF_CONSTEXPR (Size * sizeof(T) <= Alignment) {
|
||||
// Data size less than alignment will never cross multiple aligned vectors.
|
||||
v1 = v0;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
uintptr_t left_off = mem_addr & (__HVX_LENGTH__ - 1);
|
||||
if (left_off + Size * sizeof(T) > __HVX_LENGTH__) {
|
||||
v1 = HVX_vmem<1>(mem);
|
||||
@@ -99,7 +98,7 @@ EIGEN_STRONG_INLINE void HVX_store_partial(T* mem, HVX_Vector v) {
|
||||
HVX_VectorPred ql_not = Q6_Q_vsetq_R(mem_addr);
|
||||
HVX_VectorPred qr = Q6_Q_vsetq2_R(right_off);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(Size * sizeof(T) > Alignment) {
|
||||
EIGEN_IF_CONSTEXPR (Size * sizeof(T) > Alignment) {
|
||||
if (right_off > __HVX_LENGTH__) {
|
||||
Q6_vmem_QRIV(qr, mem + __HVX_LENGTH__ / sizeof(T), value);
|
||||
qr = Q6_Q_vcmp_eq_VbVb(value, value);
|
||||
|
||||
@@ -1144,13 +1144,14 @@ struct gebp_rhs_cols<J, MrPackets, NrCols, true> {
|
||||
static EIGEN_ALWAYS_INLINE void run(GEBPTraits& traits, const RhsScalar* blB, Index rhs_offset, LhsArray& A,
|
||||
RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) {
|
||||
constexpr int lane = J % 4;
|
||||
EIGEN_IF_CONSTEXPR(lane == 0)
|
||||
traits.loadRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
|
||||
else traits.updateRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
|
||||
EIGEN_IF_CONSTEXPR (lane == 0)
|
||||
traits.loadRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
|
||||
else
|
||||
traits.updateRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(MrPackets >= 1) traits.madd(A[0], rhs_panel, C[J + 0 * NrCols], T0, fix<lane>);
|
||||
EIGEN_IF_CONSTEXPR(MrPackets >= 2) traits.madd(A[1], rhs_panel, C[J + 1 * NrCols], T0, fix<lane>);
|
||||
EIGEN_IF_CONSTEXPR(MrPackets >= 3) traits.madd(A[2], rhs_panel, C[J + 2 * NrCols], T0, fix<lane>);
|
||||
EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.madd(A[0], rhs_panel, C[J + 0 * NrCols], T0, fix<lane>);
|
||||
EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.madd(A[1], rhs_panel, C[J + 1 * NrCols], T0, fix<lane>);
|
||||
EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.madd(A[2], rhs_panel, C[J + 2 * NrCols], T0, fix<lane>);
|
||||
|
||||
gebp_rhs_cols<J + 1, MrPackets, NrCols>::run(traits, blB, rhs_offset, A, rhs_panel, T0, C);
|
||||
}
|
||||
@@ -1166,9 +1167,9 @@ struct gebp_micro_step {
|
||||
RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) {
|
||||
constexpr int LhsProg = GEBPTraits::LhsProgress;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(MrPackets >= 1) traits.loadLhs(&blA[(0 + MrPackets * K) * LhsProg], A[0]);
|
||||
EIGEN_IF_CONSTEXPR(MrPackets >= 2) traits.loadLhs(&blA[(1 + MrPackets * K) * LhsProg], A[1]);
|
||||
EIGEN_IF_CONSTEXPR(MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]);
|
||||
EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.loadLhs(&blA[(0 + MrPackets * K) * LhsProg], A[0]);
|
||||
EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.loadLhs(&blA[(1 + MrPackets * K) * LhsProg], A[1]);
|
||||
EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]);
|
||||
|
||||
gebp_rhs_cols<0, MrPackets, NrCols>::run(traits, blB, Index(NrCols * K), A, rhs_panel, T0, C);
|
||||
}
|
||||
@@ -1186,7 +1187,9 @@ EIGEN_ALWAYS_INLINE void gebp_neon_3p_workaround(LhsArray_& A) {
|
||||
#if EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
|
||||
using LhsElement = std::remove_all_extents_t<std::remove_reference_t<LhsArray_>>;
|
||||
constexpr bool apply = GEBPTraits_::Vectorizable && MrPackets == 3 && std::is_same<LhsElement, FullLhsPacket_>::value;
|
||||
EIGEN_IF_CONSTEXPR(apply) { __asm__("" : "+w,m"(A[0]), "+w,m"(A[1]), "+w,m"(A[2])); }
|
||||
EIGEN_IF_CONSTEXPR (apply) {
|
||||
__asm__("" : "+w,m"(A[0]), "+w,m"(A[1]), "+w,m"(A[2]));
|
||||
}
|
||||
#else
|
||||
EIGEN_UNUSED_VARIABLE(A);
|
||||
#endif
|
||||
@@ -1204,7 +1207,7 @@ EIGEN_ALWAYS_INLINE void gebp_sse_spilling_workaround(LhsArray_& A, AccArray_& A
|
||||
using LhsElement = std::remove_all_extents_t<std::remove_reference_t<LhsArray_>>;
|
||||
constexpr bool apply =
|
||||
GEBPTraits_::Vectorizable && MrPackets <= 2 && NrCols >= 4 && std::is_same<LhsElement, FullLhsPacket_>::value;
|
||||
EIGEN_IF_CONSTEXPR(apply) {
|
||||
EIGEN_IF_CONSTEXPR (apply) {
|
||||
#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
|
||||
using AccElement = std::decay_t<decltype(ACC[0])>;
|
||||
constexpr bool pin_acc = std::is_same<AccElement, FullLhsPacket_>::value && MrPackets == 2 && NrCols == 4;
|
||||
@@ -1214,7 +1217,9 @@ EIGEN_ALWAYS_INLINE void gebp_sse_spilling_workaround(LhsArray_& A, AccArray_& A
|
||||
"+x"(ACC[7]));
|
||||
}
|
||||
#else
|
||||
EIGEN_IF_CONSTEXPR(MrPackets == 2) { __asm__("" : "+x,m"(A[0]), "+x,m"(A[1])); }
|
||||
EIGEN_IF_CONSTEXPR (MrPackets == 2) {
|
||||
__asm__("" : "+x,m"(A[0]), "+x,m"(A[1]));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
@@ -1231,7 +1236,9 @@ struct gebp_peeled_loop {
|
||||
constexpr bool use_double_accum = (MrPackets == 1 && NrCols == 4);
|
||||
|
||||
// Prefetch for 4-col paths
|
||||
EIGEN_IF_CONSTEXPR(NrCols == 4) { internal::prefetch(blB + (48 + 0)); }
|
||||
EIGEN_IF_CONSTEXPR (NrCols == 4) {
|
||||
internal::prefetch(blB + (48 + 0));
|
||||
}
|
||||
|
||||
// Helper to do one step with workarounds
|
||||
#define EIGEN_GEBP_DO_STEP(KVAL, ACC) \
|
||||
@@ -1240,7 +1247,7 @@ struct gebp_peeled_loop {
|
||||
gebp_neon_3p_workaround<MrPackets, GEBPTraits, FullLhsPacket>(A); \
|
||||
gebp_sse_spilling_workaround<MrPackets, NrCols, GEBPTraits, FullLhsPacket>(A, ACC); \
|
||||
/* LHS prefetch for 2pX4 and 3pX4 */ \
|
||||
EIGEN_IF_CONSTEXPR((MrPackets == 2 || MrPackets == 3) && NrCols == 4) { \
|
||||
EIGEN_IF_CONSTEXPR ((MrPackets == 2 || MrPackets == 3) && NrCols == 4) { \
|
||||
internal::prefetch(blA + (MrPackets * KVAL + 16) * GEBPTraits::LhsProgress); \
|
||||
if (EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) { \
|
||||
internal::prefetch(blB + (NrCols * KVAL + 16) * GEBPTraits::RhsProgress); \
|
||||
@@ -1248,23 +1255,26 @@ struct gebp_peeled_loop {
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
EIGEN_IF_CONSTEXPR(use_double_accum) {
|
||||
EIGEN_IF_CONSTEXPR (use_double_accum) {
|
||||
EIGEN_GEBP_DO_STEP(0, C);
|
||||
EIGEN_GEBP_DO_STEP(1, D);
|
||||
EIGEN_GEBP_DO_STEP(2, C);
|
||||
EIGEN_GEBP_DO_STEP(3, D);
|
||||
EIGEN_IF_CONSTEXPR(NrCols == 4) { internal::prefetch(blB + (48 + 16)); }
|
||||
EIGEN_IF_CONSTEXPR (NrCols == 4) {
|
||||
internal::prefetch(blB + (48 + 16));
|
||||
}
|
||||
EIGEN_GEBP_DO_STEP(4, C);
|
||||
EIGEN_GEBP_DO_STEP(5, D);
|
||||
EIGEN_GEBP_DO_STEP(6, C);
|
||||
EIGEN_GEBP_DO_STEP(7, D);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_GEBP_DO_STEP(0, C);
|
||||
EIGEN_GEBP_DO_STEP(1, C);
|
||||
EIGEN_GEBP_DO_STEP(2, C);
|
||||
EIGEN_GEBP_DO_STEP(3, C);
|
||||
EIGEN_IF_CONSTEXPR(NrCols == 4 && MrPackets == 2) { internal::prefetch(blB + (48 + 16)); }
|
||||
EIGEN_IF_CONSTEXPR (NrCols == 4 && MrPackets == 2) {
|
||||
internal::prefetch(blB + (48 + 16));
|
||||
}
|
||||
EIGEN_GEBP_DO_STEP(4, C);
|
||||
EIGEN_GEBP_DO_STEP(5, C);
|
||||
EIGEN_GEBP_DO_STEP(6, C);
|
||||
@@ -1321,7 +1331,7 @@ EIGEN_ALWAYS_INLINE void gebp_micro_panel_impl(GEBPTraits& traits, const DataMap
|
||||
// compiler that D[n] is always in bounds for the use_double_accum path.
|
||||
alignas(AccPacketLocal) AccPacketLocal D[CSize];
|
||||
#endif
|
||||
EIGEN_IF_CONSTEXPR(use_double_accum) {
|
||||
EIGEN_IF_CONSTEXPR (use_double_accum) {
|
||||
for (int n = 0; n < NrCols; ++n) traits.initAcc(D[n]);
|
||||
}
|
||||
|
||||
@@ -1353,7 +1363,7 @@ EIGEN_ALWAYS_INLINE void gebp_micro_panel_impl(GEBPTraits& traits, const DataMap
|
||||
}
|
||||
|
||||
// Merge double accumulators
|
||||
EIGEN_IF_CONSTEXPR(use_double_accum) {
|
||||
EIGEN_IF_CONSTEXPR (use_double_accum) {
|
||||
for (int n = 0; n < NrCols; ++n) C[n] = padd(C[n], D[n]);
|
||||
}
|
||||
|
||||
@@ -1456,7 +1466,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
|
||||
}
|
||||
|
||||
//---------- Process 3 * LhsProgress rows at once ----------
|
||||
EIGEN_IF_CONSTEXPR(mr >= 3 * Traits::LhsProgress) {
|
||||
EIGEN_IF_CONSTEXPR (mr >= 3 * Traits::LhsProgress) {
|
||||
const Index rhs_block = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar);
|
||||
const Index lhs_strip = depth * sizeof(LhsScalar) * 3 * LhsProgress;
|
||||
const Index lhs_avail = (lhs_budget > rhs_block) ? (lhs_budget - rhs_block) : 0;
|
||||
@@ -1465,7 +1475,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
|
||||
: (3 * LhsProgress) * std::max<Index>(1, lhs_avail / lhs_strip);
|
||||
for (Index i1 = 0; i1 < peeled_mc3; i1 += actual_panel_rows) {
|
||||
const Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc3);
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
for (Index i = i1; i < actual_panel_end; i += 3 * LhsProgress) {
|
||||
micro_panel(fix<3>, fix<8>, traits, i, j2);
|
||||
@@ -1486,7 +1496,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
|
||||
}
|
||||
|
||||
//---------- Process 2 * LhsProgress rows at once ----------
|
||||
EIGEN_IF_CONSTEXPR(mr >= 2 * Traits::LhsProgress) {
|
||||
EIGEN_IF_CONSTEXPR (mr >= 2 * Traits::LhsProgress) {
|
||||
const Index rhs_block2 = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar);
|
||||
const Index lhs_strip2 = depth * sizeof(LhsScalar) * 2 * LhsProgress;
|
||||
const Index lhs_avail2 = (lhs_budget > rhs_block2) ? (lhs_budget - rhs_block2) : 0;
|
||||
@@ -1496,7 +1506,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
|
||||
: (2 * LhsProgress) * std::max<Index>(1, lhs_avail2 / lhs_strip2);
|
||||
for (Index i1 = peeled_mc3; i1 < peeled_mc2; i1 += actual_panel_rows) {
|
||||
Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc2);
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
for (Index i = i1; i < actual_panel_end; i += 2 * LhsProgress) {
|
||||
micro_panel(fix<2>, fix<8>, traits, i, j2);
|
||||
@@ -1517,9 +1527,9 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
|
||||
}
|
||||
|
||||
//---------- Process 1 * LhsProgress rows at once ----------
|
||||
EIGEN_IF_CONSTEXPR(mr >= 1 * Traits::LhsProgress) {
|
||||
EIGEN_IF_CONSTEXPR (mr >= 1 * Traits::LhsProgress) {
|
||||
for (Index i = peeled_mc2; i < peeled_mc1; i += LhsProgress) {
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
micro_panel(fix<1>, fix<8>, traits, i, j2);
|
||||
}
|
||||
@@ -1534,10 +1544,10 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
|
||||
}
|
||||
|
||||
//---------- Process LhsProgressHalf rows at once ----------
|
||||
EIGEN_IF_CONSTEXPR((LhsProgressHalf < LhsProgress) && mr >= LhsProgressHalf) {
|
||||
EIGEN_IF_CONSTEXPR ((LhsProgressHalf < LhsProgress) && mr >= LhsProgressHalf) {
|
||||
HalfTraits half_traits;
|
||||
for (Index i = peeled_mc1; i < peeled_mc_half; i += LhsProgressHalf) {
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
gebp_micro_panel_impl<1, 8, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
|
||||
LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
|
||||
@@ -1558,10 +1568,10 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
|
||||
}
|
||||
|
||||
//---------- Process LhsProgressQuarter rows at once ----------
|
||||
EIGEN_IF_CONSTEXPR((LhsProgressQuarter < LhsProgressHalf) && mr >= LhsProgressQuarter) {
|
||||
EIGEN_IF_CONSTEXPR ((LhsProgressQuarter < LhsProgressHalf) && mr >= LhsProgressQuarter) {
|
||||
QuarterTraits quarter_traits;
|
||||
for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += LhsProgressQuarter) {
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
gebp_micro_panel_impl<1, 8, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
|
||||
LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
|
||||
@@ -1583,7 +1593,7 @@ EIGEN_DONT_INLINE void gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr,
|
||||
|
||||
//---------- Process remaining rows, 1 at once ----------
|
||||
if (peeled_mc_quarter < rows) {
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
// loop on each panel of the rhs
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
// loop on each row of the lhs (1*LhsProgress x depth)
|
||||
@@ -1862,9 +1872,9 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
Index i = 0;
|
||||
|
||||
// Pack 3 packets
|
||||
EIGEN_IF_CONSTEXPR(Pack1 >= 3 * PacketSize) {
|
||||
EIGEN_IF_CONSTEXPR (Pack1 >= 3 * PacketSize) {
|
||||
for (; i < peeled_mc3; i += 3 * PacketSize) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (3 * PacketSize) * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (3 * PacketSize) * offset;
|
||||
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
Packet A, B, C;
|
||||
@@ -1878,13 +1888,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
pstore(blockA + count, cj.pconj(C));
|
||||
count += PacketSize;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (3 * PacketSize) * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (3 * PacketSize) * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
// Pack 2 packets
|
||||
EIGEN_IF_CONSTEXPR(Pack1 >= 2 * PacketSize) {
|
||||
EIGEN_IF_CONSTEXPR (Pack1 >= 2 * PacketSize) {
|
||||
for (; i < peeled_mc2; i += 2 * PacketSize) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (2 * PacketSize) * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (2 * PacketSize) * offset;
|
||||
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
Packet A, B;
|
||||
@@ -1895,13 +1905,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
pstore(blockA + count, cj.pconj(B));
|
||||
count += PacketSize;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (2 * PacketSize) * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (2 * PacketSize) * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
// Pack 1 packets
|
||||
EIGEN_IF_CONSTEXPR(Pack1 >= 1 * PacketSize) {
|
||||
EIGEN_IF_CONSTEXPR (Pack1 >= 1 * PacketSize) {
|
||||
for (; i < peeled_mc1; i += 1 * PacketSize) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (1 * PacketSize) * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (1 * PacketSize) * offset;
|
||||
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
Packet A;
|
||||
@@ -1909,13 +1919,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
pstore(blockA + count, cj.pconj(A));
|
||||
count += PacketSize;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (1 * PacketSize) * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (1 * PacketSize) * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
// Pack half packets
|
||||
EIGEN_IF_CONSTEXPR(HasHalf && Pack1 >= HalfPacketSize) {
|
||||
EIGEN_IF_CONSTEXPR (HasHalf && Pack1 >= HalfPacketSize) {
|
||||
for (; i < peeled_mc_half; i += HalfPacketSize) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (HalfPacketSize)*offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (HalfPacketSize)*offset;
|
||||
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
HalfPacket A;
|
||||
@@ -1923,13 +1933,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
pstoreu(blockA + count, cj.pconj(A));
|
||||
count += HalfPacketSize;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (HalfPacketSize) * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (HalfPacketSize) * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
// Pack quarter packets
|
||||
EIGEN_IF_CONSTEXPR(HasQuarter && Pack1 >= QuarterPacketSize) {
|
||||
EIGEN_IF_CONSTEXPR (HasQuarter && Pack1 >= QuarterPacketSize) {
|
||||
for (; i < peeled_mc_quarter; i += QuarterPacketSize) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (QuarterPacketSize)*offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (QuarterPacketSize)*offset;
|
||||
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
QuarterPacket A;
|
||||
@@ -1937,7 +1947,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
pstoreu(blockA + count, cj.pconj(A));
|
||||
count += QuarterPacketSize;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (QuarterPacketSize) * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (QuarterPacketSize) * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
// Pack2 may be *smaller* than PacketSize—that happens for
|
||||
@@ -1953,23 +1963,23 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
// that case we use exactly Pack2 rows per group so the kernel's main
|
||||
// loop (which reads Pack2 = LhsProgress values via ploaddup) can
|
||||
// handle them; remaining rows fall through to the scalar loop below.
|
||||
EIGEN_IF_CONSTEXPR(Pack2 < PacketSize && Pack2 > 1) {
|
||||
EIGEN_IF_CONSTEXPR (Pack2 < PacketSize && Pack2 > 1) {
|
||||
const Index pack2_progress = (HasHalf || HasQuarter) ? last_lhs_progress : Pack2;
|
||||
const Index peeled = (HasHalf || HasQuarter) ? peeled_mc0 : (rows / Pack2) * Pack2;
|
||||
for (; i < peeled; i += pack2_progress) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += pack2_progress * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * offset;
|
||||
|
||||
for (Index k = 0; k < depth; k++)
|
||||
for (Index w = 0; w < pack2_progress; w++) blockA[count++] = cj(lhs(i + w, k));
|
||||
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += pack2_progress * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
// Pack scalars
|
||||
for (; i < rows; i++) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
|
||||
for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k));
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2012,7 +2022,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
Index peeled_mc = gone_last ? Pack2 > 1 ? (rows / pack) * pack : 0 : i + (remaining_rows / pack) * pack;
|
||||
Index starting_pos = i;
|
||||
for (; i < peeled_mc; i += pack) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += pack * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += pack * offset;
|
||||
|
||||
Index k = 0;
|
||||
if (pack >= psize && psize >= QuarterPacketSize) {
|
||||
@@ -2058,7 +2068,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
for (; w < pack; ++w) blockA[count++] = cj(lhs(i + w, k));
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += pack * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += pack * (stride - offset - depth);
|
||||
}
|
||||
|
||||
pack -= psize;
|
||||
@@ -2084,7 +2094,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
// that case we use exactly Pack2 rows per group so the kernel's main
|
||||
// loop (which reads Pack2 = LhsProgress values via ploaddup) can
|
||||
// handle them; remaining rows fall through to the scalar loop below.
|
||||
EIGEN_IF_CONSTEXPR(Pack2 < PacketSize) {
|
||||
EIGEN_IF_CONSTEXPR (Pack2 < PacketSize) {
|
||||
if (!gone_last) {
|
||||
gone_last = true;
|
||||
psize = pack = (HasHalf || HasQuarter) ? (left & ~1) : Pack2;
|
||||
@@ -2094,9 +2104,9 @@ EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Pa
|
||||
}
|
||||
|
||||
for (; i < rows; i++) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
|
||||
for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k));
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2129,10 +2139,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
|
||||
Index count = 0;
|
||||
const Index peeled_k = (depth / PacketSize) * PacketSize;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
// skip what we have before
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
|
||||
const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
|
||||
const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
|
||||
@@ -2142,10 +2152,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
|
||||
const LinearMapper dm6 = rhs.getLinearMapper(0, j2 + 6);
|
||||
const LinearMapper dm7 = rhs.getLinearMapper(0, j2 + 7);
|
||||
Index k = 0;
|
||||
EIGEN_IF_CONSTEXPR(PacketSize % 2 == 0 && PacketSize <= 8) // 2 4 8
|
||||
EIGEN_IF_CONSTEXPR (PacketSize % 2 == 0 && PacketSize <= 8) // 2 4 8
|
||||
{
|
||||
for (; k < peeled_k; k += PacketSize) {
|
||||
EIGEN_IF_CONSTEXPR(PacketSize == 2) {
|
||||
EIGEN_IF_CONSTEXPR (PacketSize == 2) {
|
||||
PacketBlock<Packet, PacketSize == 2 ? 2 : PacketSize> kernel0, kernel1, kernel2, kernel3;
|
||||
kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
|
||||
kernel0.packet[1 % PacketSize] = dm1.template loadPacket<Packet>(k);
|
||||
@@ -2170,8 +2180,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
|
||||
pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel2.packet[1 % PacketSize]));
|
||||
pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel3.packet[1 % PacketSize]));
|
||||
count += 8 * PacketSize;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(PacketSize == 4) {
|
||||
} else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
|
||||
PacketBlock<Packet, PacketSize == 4 ? 4 : PacketSize> kernel0, kernel1;
|
||||
|
||||
kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
|
||||
@@ -2194,8 +2203,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
|
||||
pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel0.packet[3 % PacketSize]));
|
||||
pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel1.packet[3 % PacketSize]));
|
||||
count += 8 * PacketSize;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(PacketSize == 8) {
|
||||
} else EIGEN_IF_CONSTEXPR (PacketSize == 8) {
|
||||
PacketBlock<Packet, PacketSize == 8 ? 8 : PacketSize> kernel0;
|
||||
|
||||
kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
|
||||
@@ -2233,28 +2241,28 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
|
||||
count += 8;
|
||||
}
|
||||
// skip what we have after
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(nr >= 4) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 4) {
|
||||
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
|
||||
// skip what we have before
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
|
||||
const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
|
||||
const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
|
||||
const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3);
|
||||
|
||||
Index k = 0;
|
||||
EIGEN_IF_CONSTEXPR((PacketSize % 4) == 0 || PacketSize == 2) {
|
||||
EIGEN_IF_CONSTEXPR ((PacketSize % 4) == 0 || PacketSize == 2) {
|
||||
for (; k < peeled_k; k += PacketSize) {
|
||||
PacketBlock<Packet, 4> kernel;
|
||||
kernel.packet[0] = dm0.template loadPacket<Packet>(k);
|
||||
kernel.packet[1] = dm1.template loadPacket<Packet>(k);
|
||||
kernel.packet[2] = dm2.template loadPacket<Packet>(k);
|
||||
kernel.packet[3] = dm3.template loadPacket<Packet>(k);
|
||||
EIGEN_IF_CONSTEXPR(PacketSize == 2) {
|
||||
EIGEN_IF_CONSTEXPR (PacketSize == 2) {
|
||||
// For PacketSize==2 we cannot ptranspose 4 packets directly; compose two
|
||||
// 2-packet transposes and re-interleave so the 4 stores produce the
|
||||
// packed-rhs layout (each store writing one half-row of the panel).
|
||||
@@ -2270,8 +2278,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
|
||||
kernel.packet[1] = tmp23.packet[0];
|
||||
kernel.packet[2] = tmp01.packet[1];
|
||||
kernel.packet[3] = tmp23.packet[1];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
ptranspose(kernel);
|
||||
}
|
||||
pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0]));
|
||||
@@ -2289,19 +2296,19 @@ EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Co
|
||||
count += 4;
|
||||
}
|
||||
// skip what we have after
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
// copy the remaining columns one at a time (nr==1)
|
||||
for (Index j2 = packet_cols4; j2 < cols; ++j2) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(0, j2);
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
blockB[count] = cj(dm0(k));
|
||||
count += 1;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2330,24 +2337,22 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, nr, RowMajor, Conjugate, PanelMo
|
||||
Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
|
||||
Index count = 0;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
|
||||
// skip what we have before
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
EIGEN_IF_CONSTEXPR(PacketSize == 8) {
|
||||
EIGEN_IF_CONSTEXPR (PacketSize == 8) {
|
||||
Packet A = rhs.template loadPacket<Packet>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
count += PacketSize;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(PacketSize == 4) {
|
||||
} else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
|
||||
Packet A = rhs.template loadPacket<Packet>(k, j2);
|
||||
Packet B = rhs.template loadPacket<Packet>(k, j2 + 4);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
pstoreu(blockB + count + PacketSize, cj.pconj(B));
|
||||
count += 2 * PacketSize;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
|
||||
blockB[count + 0] = cj(dm0(0));
|
||||
blockB[count + 1] = cj(dm0(1));
|
||||
@@ -2361,31 +2366,28 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, nr, RowMajor, Conjugate, PanelMo
|
||||
}
|
||||
}
|
||||
// skip what we have after
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(nr >= 4) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 4) {
|
||||
for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
|
||||
// skip what we have before
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
EIGEN_IF_CONSTEXPR(PacketSize == 4) {
|
||||
EIGEN_IF_CONSTEXPR (PacketSize == 4) {
|
||||
Packet A = rhs.template loadPacket<Packet>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
count += PacketSize;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 4) {
|
||||
} else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 4) {
|
||||
HalfPacket A = rhs.template loadPacket<HalfPacket>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
count += HalfPacketSize;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 4) {
|
||||
} else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 4) {
|
||||
QuarterPacket A = rhs.template loadPacket<QuarterPacket>(k, j2);
|
||||
pstoreu(blockB + count, cj.pconj(A));
|
||||
count += QuarterPacketSize;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
|
||||
blockB[count + 0] = cj(dm0(0));
|
||||
blockB[count + 1] = cj(dm0(1));
|
||||
@@ -2395,17 +2397,17 @@ struct gemm_pack_rhs<Scalar, Index, DataMapper, nr, RowMajor, Conjugate, PanelMo
|
||||
}
|
||||
}
|
||||
// skip what we have after
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * (stride - offset - depth);
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
|
||||
}
|
||||
}
|
||||
// copy the remaining columns one at a time (nr==1)
|
||||
for (Index j2 = packet_cols4; j2 < cols; ++j2) {
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += offset;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
|
||||
for (Index k = 0; k < depth; k++) {
|
||||
blockB[count] = cj(rhs(k, j2));
|
||||
count += 1;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(PanelMode) count += stride - offset - depth;
|
||||
EIGEN_IF_CONSTEXPR (PanelMode) count += stride - offset - depth;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -121,7 +121,7 @@ struct general_matrix_matrix_triangular_product<Index, LhsScalar, LhsStorageOrde
|
||||
// 1 - before the diagonal => processed with gebp or skipped
|
||||
// 2 - the actual_mc x actual_mc symmetric block => processed with a special kernel
|
||||
// 3 - after the diagonal => processed with gebp or skipped
|
||||
EIGEN_IF_CONSTEXPR(UpLo == Lower) {
|
||||
EIGEN_IF_CONSTEXPR (UpLo == Lower) {
|
||||
gebp(res.getSubMapper(i2, 0), blockA, blockB, actual_mc, actual_kc, (std::min)(size, i2), alpha, -1, -1, 0,
|
||||
0);
|
||||
}
|
||||
@@ -129,7 +129,7 @@ struct general_matrix_matrix_triangular_product<Index, LhsScalar, LhsStorageOrde
|
||||
sybb(res_ + resStride * i2 + resIncr * i2, resIncr, resStride, blockA, blockB + actual_kc * i2, actual_mc,
|
||||
actual_kc, alpha);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(UpLo == Upper) {
|
||||
EIGEN_IF_CONSTEXPR (UpLo == Upper) {
|
||||
Index j2 = i2 + actual_mc;
|
||||
gebp(res.getSubMapper(i2, j2), blockA, blockB + actual_kc * j2, actual_mc, actual_kc,
|
||||
(std::max)(Index(0), size - j2), alpha, -1, -1, 0, 0);
|
||||
@@ -171,7 +171,7 @@ struct tribb_kernel {
|
||||
Index actualBlockSize = std::min<Index>(BlockSize, size - j);
|
||||
const RhsScalar* actual_b = blockB + j * depth;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(UpLo == Upper) {
|
||||
EIGEN_IF_CONSTEXPR (UpLo == Upper) {
|
||||
gebp_kernel1(res.getSubMapper(0, j), blockA, actual_b, j, depth, actualBlockSize, alpha, -1, -1, 0, 0);
|
||||
}
|
||||
|
||||
@@ -191,7 +191,7 @@ struct tribb_kernel {
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(UpLo == Lower) {
|
||||
EIGEN_IF_CONSTEXPR (UpLo == Lower) {
|
||||
Index i = j + actualBlockSize;
|
||||
gebp_kernel1(res.getSubMapper(i, j), blockA + depth * i, actual_b, size - i, depth, actualBlockSize, alpha, -1,
|
||||
-1, 0, 0);
|
||||
@@ -240,7 +240,7 @@ struct general_product_to_triangular_selector<MatrixType, ProductType, UpLo, tru
|
||||
ei_declare_aligned_stack_constructed_variable(
|
||||
Scalar, actualLhsPtr, actualLhs.size(),
|
||||
(UseLhsDirectly ? const_cast<Scalar*>(actualLhs.data()) : static_lhs.data()));
|
||||
EIGEN_IF_CONSTEXPR(!UseLhsDirectly) {
|
||||
EIGEN_IF_CONSTEXPR (!UseLhsDirectly) {
|
||||
Map<typename ActualLhs_::PlainObject>(actualLhsPtr, actualLhs.size()) = actualLhs;
|
||||
}
|
||||
|
||||
@@ -249,7 +249,7 @@ struct general_product_to_triangular_selector<MatrixType, ProductType, UpLo, tru
|
||||
ei_declare_aligned_stack_constructed_variable(
|
||||
Scalar, actualRhsPtr, actualRhs.size(),
|
||||
(UseRhsDirectly ? const_cast<Scalar*>(actualRhs.data()) : static_rhs.data()));
|
||||
EIGEN_IF_CONSTEXPR(!UseRhsDirectly) {
|
||||
EIGEN_IF_CONSTEXPR (!UseRhsDirectly) {
|
||||
Map<typename ActualRhs_::PlainObject>(actualRhsPtr, actualRhs.size()) = actualRhs;
|
||||
}
|
||||
|
||||
|
||||
@@ -255,7 +255,7 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLh
|
||||
process_rows<1>(i, j2, jend, lhs, rhs, res, palpha, pcj);
|
||||
i += ResPacketSize;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasHalf) {
|
||||
EIGEN_IF_CONSTEXPR (HasHalf) {
|
||||
if (i < n_half) {
|
||||
ResPacketHalf c0 = pzero(ResPacketHalf{});
|
||||
for (Index j = j2; j < jend; j += 1) {
|
||||
@@ -267,7 +267,7 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLh
|
||||
i += ResPacketSizeHalf;
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasQuarter) {
|
||||
EIGEN_IF_CONSTEXPR (HasQuarter) {
|
||||
if (i < n_quarter) {
|
||||
ResPacketQuarter c0 = pzero(ResPacketQuarter{});
|
||||
for (Index j = j2; j < jend; j += 1) {
|
||||
@@ -511,14 +511,14 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLh
|
||||
c0 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i, j), b0, c0);
|
||||
}
|
||||
ResScalar cc0 = predux(c0);
|
||||
EIGEN_IF_CONSTEXPR(HasHalf) {
|
||||
EIGEN_IF_CONSTEXPR (HasHalf) {
|
||||
for (Index j = fullColBlockEnd; j < halfColBlockEnd; j += LhsPacketSizeHalf) {
|
||||
RhsPacketHalf b0 = rhs.template load<RhsPacketHalf, Unaligned>(j, 0);
|
||||
c0_h = pcj_half.pmadd(lhs.template load<LhsPacketHalf, LhsAlignment>(i, j), b0, c0_h);
|
||||
}
|
||||
cc0 += predux(c0_h);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasQuarter) {
|
||||
EIGEN_IF_CONSTEXPR (HasQuarter) {
|
||||
for (Index j = halfColBlockEnd; j < quarterColBlockEnd; j += LhsPacketSizeQuarter) {
|
||||
RhsPacketQuarter b0 = rhs.template load<RhsPacketQuarter, Unaligned>(j, 0);
|
||||
c0_q = pcj_quarter.pmadd(lhs.template load<LhsPacketQuarter, LhsAlignment>(i, j), b0, c0_q);
|
||||
@@ -631,7 +631,7 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLh
|
||||
using Unroll = gemv_small_cols_unroller<N - 1, N>;
|
||||
|
||||
ResScalar cc[N] = {};
|
||||
EIGEN_IF_CONSTEXPR(HasHalf) {
|
||||
EIGEN_IF_CONSTEXPR (HasHalf) {
|
||||
ResPacketHalf h[N];
|
||||
Unroll::init_zero(h);
|
||||
for (Index j = 0; j < halfColBlockEnd; j += LhsPacketSizeHalf) {
|
||||
@@ -640,7 +640,7 @@ general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLh
|
||||
}
|
||||
Unroll::predux_accum(cc, h);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasQuarter) {
|
||||
EIGEN_IF_CONSTEXPR (HasQuarter) {
|
||||
ResPacketQuarter q[N];
|
||||
Unroll::init_zero(q);
|
||||
for (Index j = halfColBlockEnd; j < quarterColBlockEnd; j += LhsPacketSizeQuarter) {
|
||||
|
||||
@@ -69,22 +69,22 @@ struct symm_pack_lhs {
|
||||
? peeled_mc_half + ((rows - peeled_mc_half) / (QuarterPacketSize)) * (QuarterPacketSize)
|
||||
: 0;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(Pack1 >= 3 * PacketSize)
|
||||
for (Index i = 0; i < peeled_mc3; i += 3 * PacketSize) pack<3 * PacketSize>(blockA, lhs, cols, i, count);
|
||||
EIGEN_IF_CONSTEXPR (Pack1 >= 3 * PacketSize)
|
||||
for (Index i = 0; i < peeled_mc3; i += 3 * PacketSize) pack<3 * PacketSize>(blockA, lhs, cols, i, count);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(Pack1 >= 2 * PacketSize)
|
||||
for (Index i = peeled_mc3; i < peeled_mc2; i += 2 * PacketSize) pack<2 * PacketSize>(blockA, lhs, cols, i, count);
|
||||
EIGEN_IF_CONSTEXPR (Pack1 >= 2 * PacketSize)
|
||||
for (Index i = peeled_mc3; i < peeled_mc2; i += 2 * PacketSize) pack<2 * PacketSize>(blockA, lhs, cols, i, count);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(Pack1 >= 1 * PacketSize)
|
||||
for (Index i = peeled_mc2; i < peeled_mc1; i += 1 * PacketSize) pack<1 * PacketSize>(blockA, lhs, cols, i, count);
|
||||
EIGEN_IF_CONSTEXPR (Pack1 >= 1 * PacketSize)
|
||||
for (Index i = peeled_mc2; i < peeled_mc1; i += 1 * PacketSize) pack<1 * PacketSize>(blockA, lhs, cols, i, count);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(HasHalf && Pack1 >= HalfPacketSize)
|
||||
for (Index i = peeled_mc1; i < peeled_mc_half; i += HalfPacketSize)
|
||||
pack<HalfPacketSize>(blockA, lhs, cols, i, count);
|
||||
EIGEN_IF_CONSTEXPR (HasHalf && Pack1 >= HalfPacketSize)
|
||||
for (Index i = peeled_mc1; i < peeled_mc_half; i += HalfPacketSize)
|
||||
pack<HalfPacketSize>(blockA, lhs, cols, i, count);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(HasQuarter && Pack1 >= QuarterPacketSize)
|
||||
for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += QuarterPacketSize)
|
||||
pack<QuarterPacketSize>(blockA, lhs, cols, i, count);
|
||||
EIGEN_IF_CONSTEXPR (HasQuarter && Pack1 >= QuarterPacketSize)
|
||||
for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += QuarterPacketSize)
|
||||
pack<QuarterPacketSize>(blockA, lhs, cols, i, count);
|
||||
|
||||
// do the same with mr==1
|
||||
for (Index i = peeled_mc_quarter; i < rows; i++) {
|
||||
@@ -112,11 +112,11 @@ struct symm_pack_rhs {
|
||||
for (Index k = k2; k < end_k; k++) {
|
||||
blockB[count + 0] = rhs(k, j2 + 0);
|
||||
blockB[count + 1] = rhs(k, j2 + 1);
|
||||
EIGEN_IF_CONSTEXPR(nr >= 4) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 4) {
|
||||
blockB[count + 2] = rhs(k, j2 + 2);
|
||||
blockB[count + 3] = rhs(k, j2 + 3);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
blockB[count + 4] = rhs(k, j2 + 4);
|
||||
blockB[count + 5] = rhs(k, j2 + 5);
|
||||
blockB[count + 6] = rhs(k, j2 + 6);
|
||||
@@ -128,7 +128,7 @@ struct symm_pack_rhs {
|
||||
|
||||
// second part: diagonal block
|
||||
Index end8 = nr >= 8 ? (std::min)(k2 + rows, packet_cols8) : k2;
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = k2; j2 < end8; j2 += 8) {
|
||||
// again we can split vertically in three different parts (transpose, symmetric, normal)
|
||||
// transpose
|
||||
@@ -170,7 +170,7 @@ struct symm_pack_rhs {
|
||||
}
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(nr >= 4) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 4) {
|
||||
for (Index j2 = end8; j2 < (std::min)(k2 + rows, packet_cols4); j2 += 4) {
|
||||
// again we can split vertically in three different parts (transpose, symmetric, normal)
|
||||
// transpose
|
||||
@@ -206,7 +206,7 @@ struct symm_pack_rhs {
|
||||
}
|
||||
|
||||
// third part: transposed
|
||||
EIGEN_IF_CONSTEXPR(nr >= 8) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 8) {
|
||||
for (Index j2 = k2 + rows; j2 < packet_cols8; j2 += 8) {
|
||||
for (Index k = k2; k < end_k; k++) {
|
||||
blockB[count + 0] = numext::conj(rhs(j2 + 0, k));
|
||||
@@ -221,7 +221,7 @@ struct symm_pack_rhs {
|
||||
}
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(nr >= 4) {
|
||||
EIGEN_IF_CONSTEXPR (nr >= 4) {
|
||||
for (Index j2 = (std::max)(packet_cols8, k2 + rows); j2 < packet_cols4; j2 += 4) {
|
||||
for (Index k = k2; k < end_k; k++) {
|
||||
blockB[count + 0] = numext::conj(rhs(j2 + 0, k));
|
||||
|
||||
@@ -57,11 +57,10 @@ struct selfadjoint_matrix_vector_product_symv
|
||||
Specialized> { \
|
||||
static void run(Index size, const Scalar* lhs, Index lhsStride, const Scalar* _rhs, Scalar* res, Scalar alpha) { \
|
||||
enum { IsColMajor = StorageOrder == ColMajor }; \
|
||||
EIGEN_IF_CONSTEXPR(IsColMajor == ConjugateLhs) { \
|
||||
EIGEN_IF_CONSTEXPR (IsColMajor == ConjugateLhs) { \
|
||||
selfadjoint_matrix_vector_product<Scalar, Index, StorageOrder, UpLo, ConjugateLhs, ConjugateRhs, \
|
||||
BuiltIn>::run(size, lhs, lhsStride, _rhs, res, alpha); \
|
||||
} \
|
||||
else { \
|
||||
} else { \
|
||||
selfadjoint_matrix_vector_product_symv<Scalar, Index, StorageOrder, UpLo, ConjugateLhs, ConjugateRhs>::run( \
|
||||
size, lhs, lhsStride, _rhs, res, alpha); \
|
||||
} \
|
||||
|
||||
@@ -40,7 +40,7 @@ struct selfadjoint_rank1_update<Scalar, Index, ColMajor, UpLo, ConjLhs, ConjRhs>
|
||||
Packet ps0 = internal::pset1<Packet>(s0);
|
||||
Packet ps1 = internal::pset1<Packet>(s1);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(UpLo == Lower) {
|
||||
EIGEN_IF_CONSTEXPR (UpLo == Lower) {
|
||||
Scalar* EIGEN_RESTRICT col0 = mat + stride * j + j;
|
||||
Scalar* EIGEN_RESTRICT col1 = mat + stride * (j + 1) + (j + 1);
|
||||
|
||||
@@ -71,8 +71,7 @@ struct selfadjoint_rank1_update<Scalar, Index, ColMajor, UpLo, ConjLhs, ConjRhs>
|
||||
d0[k] += s0 * cx;
|
||||
d1[k] += s1 * cx;
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// UpLo == Upper
|
||||
Scalar* EIGEN_RESTRICT col0 = mat + stride * j;
|
||||
Scalar* EIGEN_RESTRICT col1 = mat + stride * (j + 1);
|
||||
@@ -162,7 +161,7 @@ struct selfadjoint_product_selector<MatrixType, OtherType, UpLo, true> {
|
||||
Scalar, actualOtherPtr, other.size(),
|
||||
(UseOtherDirectly ? const_cast<Scalar*>(actualOther.data()) : static_other.data()));
|
||||
|
||||
EIGEN_IF_CONSTEXPR(!UseOtherDirectly) {
|
||||
EIGEN_IF_CONSTEXPR (!UseOtherDirectly) {
|
||||
Map<typename ActualOtherType_::PlainObject>(actualOtherPtr, actualOther.size()) = actualOther;
|
||||
}
|
||||
|
||||
|
||||
@@ -220,7 +220,9 @@ EIGEN_DEVICE_FUNC SelfAdjointView<MatrixType, UpLo>& SelfAdjointView<MatrixType,
|
||||
|
||||
Scalar actualAlpha = alpha * UBlasTraits::extractScalarFactor(u.derived()) *
|
||||
numext::conj(VBlasTraits::extractScalarFactor(v.derived()));
|
||||
EIGEN_IF_CONSTEXPR(IsRowMajor) { actualAlpha = numext::conj(actualAlpha); }
|
||||
EIGEN_IF_CONSTEXPR (IsRowMajor) {
|
||||
actualAlpha = numext::conj(actualAlpha);
|
||||
}
|
||||
|
||||
const Index size = u.size();
|
||||
|
||||
@@ -229,9 +231,10 @@ EIGEN_DEVICE_FUNC SelfAdjointView<MatrixType, UpLo>& SelfAdjointView<MatrixType,
|
||||
static_u;
|
||||
ei_declare_aligned_stack_constructed_variable(Scalar, uPtr, size,
|
||||
(UseUDirectly ? const_cast<Scalar*>(actualU.data()) : static_u.data()));
|
||||
EIGEN_IF_CONSTEXPR(!UseUDirectly) {
|
||||
EIGEN_IF_CONSTEXPR(NeedConjU) { Map<typename ActualUType_::PlainObject>(uPtr, size) = actualU.conjugate(); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (!UseUDirectly) {
|
||||
EIGEN_IF_CONSTEXPR (NeedConjU) {
|
||||
Map<typename ActualUType_::PlainObject>(uPtr, size) = actualU.conjugate();
|
||||
} else {
|
||||
Map<typename ActualUType_::PlainObject>(uPtr, size) = actualU;
|
||||
}
|
||||
}
|
||||
@@ -241,9 +244,10 @@ EIGEN_DEVICE_FUNC SelfAdjointView<MatrixType, UpLo>& SelfAdjointView<MatrixType,
|
||||
static_v;
|
||||
ei_declare_aligned_stack_constructed_variable(Scalar, vPtr, size,
|
||||
(UseVDirectly ? const_cast<Scalar*>(actualV.data()) : static_v.data()));
|
||||
EIGEN_IF_CONSTEXPR(!UseVDirectly) {
|
||||
EIGEN_IF_CONSTEXPR(NeedConjV) { Map<typename ActualVType_::PlainObject>(vPtr, size) = actualV.conjugate(); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (!UseVDirectly) {
|
||||
EIGEN_IF_CONSTEXPR (NeedConjV) {
|
||||
Map<typename ActualVType_::PlainObject>(vPtr, size) = actualV.conjugate();
|
||||
} else {
|
||||
Map<typename ActualVType_::PlainObject>(vPtr, size) = actualV;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,9 +92,10 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix<
|
||||
|
||||
Matrix<Scalar, SmallPanelWidth, SmallPanelWidth, LhsStorageOrder> triangularBuffer;
|
||||
triangularBuffer.setZero();
|
||||
EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag)
|
||||
triangularBuffer.diagonal().setZero();
|
||||
else triangularBuffer.diagonal().setOnes();
|
||||
EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag)
|
||||
triangularBuffer.diagonal().setZero();
|
||||
else
|
||||
triangularBuffer.diagonal().setOnes();
|
||||
|
||||
gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
|
||||
gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing,
|
||||
@@ -217,9 +218,10 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix<
|
||||
|
||||
Matrix<Scalar, SmallPanelWidth, SmallPanelWidth, RhsStorageOrder> triangularBuffer;
|
||||
triangularBuffer.setZero();
|
||||
EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag)
|
||||
triangularBuffer.diagonal().setZero();
|
||||
else triangularBuffer.diagonal().setOnes();
|
||||
EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag)
|
||||
triangularBuffer.diagonal().setZero();
|
||||
else
|
||||
triangularBuffer.diagonal().setOnes();
|
||||
|
||||
gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
|
||||
gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing,
|
||||
@@ -233,7 +235,7 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix<
|
||||
Index actual_k2 = IsLower ? k2 : k2 - actual_kc;
|
||||
|
||||
// align blocks with the end of the triangular part for trapezoidal rhs
|
||||
EIGEN_IF_CONSTEXPR(IsLower) {
|
||||
EIGEN_IF_CONSTEXPR (IsLower) {
|
||||
if ((k2 < cols) && (actual_k2 + actual_kc > cols)) {
|
||||
actual_kc = cols - k2;
|
||||
k2 = actual_k2 + actual_kc - kc;
|
||||
@@ -355,7 +357,7 @@ struct triangular_product_impl<Mode, LhsIsTriangular, Lhs, false, Rhs, false> {
|
||||
actualAlpha, blocking);
|
||||
|
||||
// Apply correction if the diagonal is unit and a scalar factor was nested:
|
||||
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) {
|
||||
if (LhsIsTriangular && !numext::is_exactly_one(lhs_alpha)) {
|
||||
Index diagSize = (std::min)(lhs.rows(), lhs.cols());
|
||||
dst.topRows(diagSize) -= ((lhs_alpha - LhsScalar(1)) * a_rhs).topRows(diagSize);
|
||||
|
||||
@@ -55,7 +55,7 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
|
||||
|
||||
// Process the triangular panel using raw pointer operations with 2-column batching
|
||||
// to eliminate expression template overhead and share result loads/stores.
|
||||
EIGEN_IF_CONSTEXPR(IsLower) {
|
||||
EIGEN_IF_CONSTEXPR (IsLower) {
|
||||
Index k = 0;
|
||||
for (; k + 1 < actualPanelWidth; k += 2) {
|
||||
Index i0 = pi + k;
|
||||
@@ -66,18 +66,18 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
|
||||
const LhsScalar* EIGEN_RESTRICT c1 = lhs_ + i1 * lhsStride;
|
||||
|
||||
// Diagonal of column 0
|
||||
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) res_[i0] += s0 * cjl(c0[i0]);
|
||||
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) res_[i0] += s0 * cjl(c0[i0]);
|
||||
// Row i1: contribution from column 0 + diagonal of column 1
|
||||
{
|
||||
ResScalar r1 = s0 * cjl(c0[i1]);
|
||||
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) r1 += s1 * cjl(c1[i1]);
|
||||
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) r1 += s1 * cjl(c1[i1]);
|
||||
res_[i1] += r1;
|
||||
}
|
||||
// Shared rows where both columns contribute
|
||||
Index panelEnd = pi + actualPanelWidth;
|
||||
for (Index j = i1 + 1; j < panelEnd; ++j) res_[j] += s0 * cjl(c0[j]) + s1 * cjl(c1[j]);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
|
||||
res_[i0] += s0;
|
||||
res_[i1] += s1;
|
||||
}
|
||||
@@ -86,11 +86,10 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
|
||||
Index i = pi + k;
|
||||
ResScalar s = alpha * cjr(rhs_[i * rhsIncr]);
|
||||
const LhsScalar* EIGEN_RESTRICT c = lhs_ + i * lhsStride;
|
||||
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) res_[i] += s * cjl(c[i]);
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) res_[i] += s;
|
||||
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) res_[i] += s * cjl(c[i]);
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) res_[i] += s;
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// Upper triangular: process 2 columns at a time
|
||||
Index k = 0;
|
||||
for (; k + 1 < actualPanelWidth; k += 2) {
|
||||
@@ -107,13 +106,13 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
|
||||
// Row i0: diagonal of col0 + contribution from col1
|
||||
{
|
||||
ResScalar r0 = s1 * cjl(c1[i0]);
|
||||
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) r0 += s0 * cjl(c0[i0]);
|
||||
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) r0 += s0 * cjl(c0[i0]);
|
||||
res_[i0] += r0;
|
||||
}
|
||||
// Diagonal of column 1
|
||||
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) res_[i1] += s1 * cjl(c1[i1]);
|
||||
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) res_[i1] += s1 * cjl(c1[i1]);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
|
||||
res_[i0] += s0;
|
||||
res_[i1] += s1;
|
||||
}
|
||||
@@ -123,8 +122,8 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
|
||||
ResScalar s = alpha * cjr(rhs_[i * rhsIncr]);
|
||||
const LhsScalar* EIGEN_RESTRICT c = lhs_ + i * lhsStride;
|
||||
for (Index j = pi; j < i; ++j) res_[j] += s * cjl(c[j]);
|
||||
EIGEN_IF_CONSTEXPR(!(HasUnitDiag || HasZeroDiag)) res_[i] += s * cjl(c[i]);
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) res_[i] += s;
|
||||
EIGEN_IF_CONSTEXPR (!(HasUnitDiag || HasZeroDiag)) res_[i] += s * cjl(c[i]);
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) res_[i] += s;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,7 +136,7 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
|
||||
RhsMapper(&rhs_[pi * rhsIncr], rhsIncr), &res_[s], resIncr, alpha);
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(!IsLower) {
|
||||
EIGEN_IF_CONSTEXPR (!IsLower) {
|
||||
if (cols > size) {
|
||||
general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjLhs, RhsScalar, RhsMapper, ConjRhs>::run(
|
||||
rows, cols - size, LhsMapper(&lhs_[size * lhsStride], lhsStride), RhsMapper(&rhs_[size * rhsIncr], rhsIncr),
|
||||
@@ -184,18 +183,17 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
|
||||
const LhsScalar* EIGEN_RESTRICT row_i = lhs_ + i * lhsStride;
|
||||
ResScalar dot = ResScalar(0);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(IsLower) {
|
||||
EIGEN_IF_CONSTEXPR (IsLower) {
|
||||
Index s = pi;
|
||||
Index len = (HasUnitDiag || HasZeroDiag) ? k : k + 1;
|
||||
for (Index j = 0; j < len; ++j) dot += cjl(row_i[s + j]) * cjr(rhs_[s + j]);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
Index s = (HasUnitDiag || HasZeroDiag) ? i + 1 : i;
|
||||
Index len = pi + actualPanelWidth - s;
|
||||
for (Index j = 0; j < len; ++j) dot += cjl(row_i[s + j]) * cjr(rhs_[s + j]);
|
||||
}
|
||||
res_[i * resIncr] += alpha * dot;
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) res_[i * resIncr] += alpha * cjr(rhs_[i]);
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) res_[i * resIncr] += alpha * cjr(rhs_[i]);
|
||||
}
|
||||
|
||||
// Rectangular part: delegate to optimized GEMV
|
||||
@@ -207,7 +205,7 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index, Mode, LhsScalar,
|
||||
RhsMapper(&rhs_[s], rhsIncr), &res_[pi * resIncr], resIncr, alpha);
|
||||
}
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(IsLower) {
|
||||
EIGEN_IF_CONSTEXPR (IsLower) {
|
||||
if (rows > diagSize) {
|
||||
general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjLhs, RhsScalar, RhsMapper, ConjRhs>::run(
|
||||
rows - diagSize, cols, LhsMapper(&lhs_[diagSize * lhsStride], lhsStride), RhsMapper(rhs_, rhsIncr),
|
||||
|
||||
@@ -98,7 +98,7 @@ EIGEN_STRONG_INLINE void trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageO
|
||||
Index j = IsLower ? size - k - 1 : k;
|
||||
|
||||
typename LhsMapper::LinearMapper r = lhs.getLinearMapper(0, j);
|
||||
EIGEN_IF_CONSTEXPR(OtherInnerStride == 1 && packet_traits<Scalar>::Vectorizable) {
|
||||
EIGEN_IF_CONSTEXPR (OtherInnerStride == 1 && packet_traits<Scalar>::Vectorizable) {
|
||||
using Packet = typename packet_traits<Scalar>::type;
|
||||
constexpr Index PS = unpacket_traits<Packet>::size;
|
||||
// Unrolled k3 loop by 4 to reduce r load/store traffic.
|
||||
@@ -144,7 +144,7 @@ EIGEN_STRONG_INLINE void trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageO
|
||||
for (; i < otherSize; ++i) r(i) -= a(i) * b;
|
||||
}
|
||||
// Vectorized diagonal scaling.
|
||||
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == 0) {
|
||||
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) {
|
||||
Scalar inv_rjj = RealScalar(1) / conj(rhs(j, j));
|
||||
Packet pinv = pset1<Packet>(inv_rjj);
|
||||
Index i = 0;
|
||||
@@ -153,14 +153,13 @@ EIGEN_STRONG_INLINE void trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageO
|
||||
}
|
||||
for (; i < otherSize; ++i) r(i) *= inv_rjj;
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (Index k3 = 0; k3 < k; ++k3) {
|
||||
Scalar b = conj(rhs(IsLower ? j + 1 + k3 : k3, j));
|
||||
typename LhsMapper::LinearMapper a = lhs.getLinearMapper(0, IsLower ? j + 1 + k3 : k3);
|
||||
for (Index i = 0; i < otherSize; ++i) r(i) -= a(i) * b;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == 0) {
|
||||
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) {
|
||||
Scalar inv_rjj = RealScalar(1) / conj(rhs(j, j));
|
||||
for (Index i = 0; i < otherSize; ++i) r(i) *= inv_rjj;
|
||||
}
|
||||
@@ -202,8 +201,8 @@ EIGEN_DONT_INLINE void triangular_solve_matrix<Scalar, Index, OnTheLeft, Mode, C
|
||||
|
||||
#if defined(EIGEN_VECTORIZE_AVX512) && defined(EIGEN_USE_AVX512_TRSM_L_KERNELS) && EIGEN_USE_AVX512_TRSM_L_KERNELS && \
|
||||
EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS
|
||||
EIGEN_IF_CONSTEXPR(
|
||||
(OtherInnerStride == 1 && (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
|
||||
EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 &&
|
||||
(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
|
||||
// Very rough cutoffs to determine when to call trsm w/o packing
|
||||
// For small problem sizes trsmKernel compiled with clang is generally faster.
|
||||
// TODO: Investigate better heuristics for cutoffs.
|
||||
@@ -270,8 +269,8 @@ EIGEN_DONT_INLINE void triangular_solve_matrix<Scalar, Index, OnTheLeft, Mode, C
|
||||
{
|
||||
Index i = IsLower ? k2 + k1 : k2 - k1;
|
||||
#if defined(EIGEN_VECTORIZE_AVX512) && defined(EIGEN_USE_AVX512_TRSM_L_KERNELS) && EIGEN_USE_AVX512_TRSM_L_KERNELS
|
||||
EIGEN_IF_CONSTEXPR(
|
||||
(OtherInnerStride == 1 && (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
|
||||
EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 &&
|
||||
(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
|
||||
i = IsLower ? k2 + k1 : k2 - k1 - actualPanelWidth;
|
||||
}
|
||||
#endif
|
||||
@@ -335,8 +334,8 @@ EIGEN_DONT_INLINE void triangular_solve_matrix<Scalar, Index, OnTheRight, Mode,
|
||||
|
||||
#if defined(EIGEN_VECTORIZE_AVX512) && defined(EIGEN_USE_AVX512_TRSM_R_KERNELS) && EIGEN_USE_AVX512_TRSM_R_KERNELS && \
|
||||
EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS
|
||||
EIGEN_IF_CONSTEXPR(
|
||||
(OtherInnerStride == 1 && (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
|
||||
EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 &&
|
||||
(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
|
||||
// TODO: Investigate better heuristics for cutoffs.
|
||||
std::ptrdiff_t l1, l2, l3;
|
||||
manage_caching_sizes(GetAction, &l1, &l2, &l3);
|
||||
|
||||
@@ -494,7 +494,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) {
|
||||
if (size == 0) return nullptr; // short-cut. Also fixes Bug 884
|
||||
check_size_for_overflow<T>(size);
|
||||
T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * size));
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
|
||||
EIGEN_TRY { default_construct_elements_of_array(result, size); }
|
||||
EIGEN_CATCH(...) {
|
||||
conditional_aligned_free<Align>(result);
|
||||
@@ -506,7 +506,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) {
|
||||
|
||||
template <typename T, bool Align>
|
||||
EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) {
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
|
||||
return conditional_aligned_realloc_new<T, Align>(pts, new_size, old_size);
|
||||
}
|
||||
|
||||
@@ -518,7 +518,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::si
|
||||
|
||||
template <typename T, bool Align>
|
||||
EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T* ptr, std::size_t size) {
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) destruct_elements_of_array<T>(ptr, size);
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) destruct_elements_of_array<T>(ptr, size);
|
||||
conditional_aligned_free<Align>(ptr);
|
||||
}
|
||||
|
||||
@@ -673,12 +673,12 @@ class aligned_stack_memory_handler {
|
||||
**/
|
||||
EIGEN_DEVICE_FUNC aligned_stack_memory_handler(T* ptr, std::size_t size, bool dealloc)
|
||||
: m_ptr(ptr), m_size(size), m_deallocate(dealloc) {
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
|
||||
if (m_ptr) Eigen::internal::default_construct_elements_of_array(m_ptr, size);
|
||||
}
|
||||
}
|
||||
EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() {
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<T>::RequireInitialization) {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) {
|
||||
if (m_ptr) Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size);
|
||||
}
|
||||
if (m_deallocate) Eigen::internal::aligned_free(m_ptr);
|
||||
@@ -718,14 +718,14 @@ struct local_nested_eval_wrapper<Xpr, NbEvaluations, true> {
|
||||
: object(ptr == 0 ? reinterpret_cast<Scalar*>(Eigen::internal::aligned_malloc(sizeof(Scalar) * xpr.size())) : ptr,
|
||||
xpr.rows(), xpr.cols()),
|
||||
m_deallocate(ptr == 0) {
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::RequireInitialization) {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
|
||||
if (object.data()) Eigen::internal::default_construct_elements_of_array(object.data(), object.size());
|
||||
}
|
||||
object = xpr;
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC ~local_nested_eval_wrapper() {
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::RequireInitialization) {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::RequireInitialization) {
|
||||
if (object.data()) Eigen::internal::destruct_elements_of_array(object.data(), object.size());
|
||||
}
|
||||
if (m_deallocate) Eigen::internal::aligned_free(object.data());
|
||||
|
||||
@@ -157,13 +157,12 @@ VectorwiseOp<ExpressionType, Direction>::cross(const MatrixBase<OtherDerived>& o
|
||||
typename internal::nested_eval<OtherDerived, 2>::type vec(other.derived());
|
||||
|
||||
CrossReturnType res(_expression().rows(), _expression().cols());
|
||||
EIGEN_IF_CONSTEXPR(Direction == Vertical) {
|
||||
EIGEN_IF_CONSTEXPR (Direction == Vertical) {
|
||||
eigen_assert(CrossReturnType::RowsAtCompileTime == 3 && "the matrix must have exactly 3 rows");
|
||||
res.row(0) = (mat.row(1) * vec.coeff(2) - mat.row(2) * vec.coeff(1)).conjugate();
|
||||
res.row(1) = (mat.row(2) * vec.coeff(0) - mat.row(0) * vec.coeff(2)).conjugate();
|
||||
res.row(2) = (mat.row(0) * vec.coeff(1) - mat.row(1) * vec.coeff(0)).conjugate();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
eigen_assert(CrossReturnType::ColsAtCompileTime == 3 && "the matrix must have exactly 3 columns");
|
||||
res.col(0) = (mat.col(1) * vec.coeff(2) - mat.col(2) * vec.coeff(1)).conjugate();
|
||||
res.col(1) = (mat.col(2) * vec.coeff(0) - mat.col(0) * vec.coeff(2)).conjugate();
|
||||
|
||||
@@ -1429,7 +1429,7 @@ struct transform_left_product_impl<Other, Mode, Options, Dim, HDim, Dim, Dim> {
|
||||
typedef TransformType ResultType;
|
||||
static EIGEN_DEVICE_FUNC ResultType run(const Other& other, const TransformType& tr) {
|
||||
TransformType res;
|
||||
EIGEN_IF_CONSTEXPR(Mode != int(AffineCompact)) res.matrix().row(Dim) = tr.matrix().row(Dim);
|
||||
EIGEN_IF_CONSTEXPR (Mode != int(AffineCompact)) res.matrix().row(Dim) = tr.matrix().row(Dim);
|
||||
res.matrix().template topRows<Dim>().noalias() = other * tr.matrix().template topRows<Dim>();
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -142,7 +142,7 @@ class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner<Scalar_>
|
||||
LeastSquareDiagonalPreconditioner& factorize(const MatType& mat) {
|
||||
// Compute the inverse squared-norm of each column of mat
|
||||
m_invdiag.resize(mat.cols());
|
||||
EIGEN_IF_CONSTEXPR(MatType::IsRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (MatType::IsRowMajor) {
|
||||
m_invdiag.setZero();
|
||||
for (Index j = 0; j < mat.outerSize(); ++j) {
|
||||
for (typename MatType::InnerIterator it(mat, j); it; ++it) m_invdiag(it.index()) += numext::abs2(it.value());
|
||||
@@ -151,8 +151,7 @@ class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner<Scalar_>
|
||||
RealScalar sum = numext::real(m_invdiag(j));
|
||||
m_invdiag(j) = sum > RealScalar(0) ? RealScalar(1) / sum : RealScalar(1);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (Index j = 0; j < mat.outerSize(); ++j) {
|
||||
RealScalar sum = mat.col(j).squaredNorm();
|
||||
m_invdiag(j) = sum > RealScalar(0) ? RealScalar(1) / sum : RealScalar(1);
|
||||
|
||||
+22
-24
@@ -785,31 +785,29 @@ JacobiSVD<MatrixType, Options>& JacobiSVD<MatrixType, Options>::compute_impl(con
|
||||
// Non-blocking paths: apply rotations individually. The real and complex
|
||||
// paths are kept separate to avoid any codegen impact from the complex
|
||||
// preconditioner on GCC's optimization of the real inner loop.
|
||||
else
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex) {
|
||||
// Complex non-blocking sweep: condition each 2x2 block to be real before diagonalizing.
|
||||
for (Index p = 1; p < n; ++p) {
|
||||
for (Index q = 0; q < p; ++q) {
|
||||
RealScalar threshold = numext::maxi<RealScalar>(considerAsZero, precision * maxDiagEntry);
|
||||
if (abs(m_workMatrix.coeff(p, q)) > threshold || abs(m_workMatrix.coeff(q, p)) > threshold) {
|
||||
finished = false;
|
||||
if (internal::svd_precondition_2x2_block_to_be_real<MatrixType, Options>::run(m_workMatrix, *this, p, q,
|
||||
maxDiagEntry)) {
|
||||
JacobiRotation<RealScalar> j_left, j_right;
|
||||
internal::real_2x2_jacobi_svd(m_workMatrix, p, q, &j_left, &j_right);
|
||||
m_workMatrix.applyOnTheLeft(p, q, j_left);
|
||||
if (computeU()) m_matrixU.applyOnTheRight(p, q, j_left.transpose());
|
||||
m_workMatrix.applyOnTheRight(p, q, j_right);
|
||||
if (computeV()) m_matrixV.applyOnTheRight(p, q, j_right);
|
||||
maxDiagEntry = numext::maxi<RealScalar>(
|
||||
maxDiagEntry,
|
||||
numext::maxi<RealScalar>(abs(m_workMatrix.coeff(p, p)), abs(m_workMatrix.coeff(q, q))));
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
|
||||
// Complex non-blocking sweep: condition each 2x2 block to be real before diagonalizing.
|
||||
for (Index p = 1; p < n; ++p) {
|
||||
for (Index q = 0; q < p; ++q) {
|
||||
RealScalar threshold = numext::maxi<RealScalar>(considerAsZero, precision * maxDiagEntry);
|
||||
if (abs(m_workMatrix.coeff(p, q)) > threshold || abs(m_workMatrix.coeff(q, p)) > threshold) {
|
||||
finished = false;
|
||||
if (internal::svd_precondition_2x2_block_to_be_real<MatrixType, Options>::run(m_workMatrix, *this, p, q,
|
||||
maxDiagEntry)) {
|
||||
JacobiRotation<RealScalar> j_left, j_right;
|
||||
internal::real_2x2_jacobi_svd(m_workMatrix, p, q, &j_left, &j_right);
|
||||
m_workMatrix.applyOnTheLeft(p, q, j_left);
|
||||
if (computeU()) m_matrixU.applyOnTheRight(p, q, j_left.transpose());
|
||||
m_workMatrix.applyOnTheRight(p, q, j_right);
|
||||
if (computeV()) m_matrixV.applyOnTheRight(p, q, j_right);
|
||||
maxDiagEntry = numext::maxi<RealScalar>(
|
||||
maxDiagEntry,
|
||||
numext::maxi<RealScalar>(abs(m_workMatrix.coeff(p, p)), abs(m_workMatrix.coeff(q, q))));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// Real non-blocking sweep: diagonalize each 2x2 block directly.
|
||||
RealScalar threshold = numext::maxi<RealScalar>(considerAsZero, precision * maxDiagEntry);
|
||||
for (Index p = 1; p < n; ++p) {
|
||||
@@ -840,7 +838,7 @@ JacobiSVD<MatrixType, Options>& JacobiSVD<MatrixType, Options>::compute_impl(con
|
||||
// treated by svd_precondition_2x2_block_to_be_real, and the imaginary part
|
||||
// of some diagonal entry might not be null.
|
||||
bool diagonal_has_imaginary_part = false;
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex) {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
|
||||
diagonal_has_imaginary_part = abs(numext::imag(m_workMatrix.coeff(i, i))) > considerAsZero;
|
||||
}
|
||||
if (diagonal_has_imaginary_part) {
|
||||
@@ -952,7 +950,7 @@ EIGEN_DONT_INLINE bool JacobiSVD<MatrixType, Options>::blocked_sweep(RealScalar
|
||||
// Right operations (column scaling) are applied directly since column
|
||||
// ops are contiguous in column-major layout.
|
||||
bool doRealSvd = true;
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex) {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
|
||||
Scalar z;
|
||||
// nn = ||(w_pp, w_qp)||_2, the norm of the first column of the 2x2 block.
|
||||
RealScalar nn = sqrt(numext::abs2(blockBuffer.coeff(kBlockSize, kBlockSize)) +
|
||||
@@ -1066,7 +1064,7 @@ EIGEN_DONT_INLINE bool JacobiSVD<MatrixType, Options>::blocked_sweep(RealScalar
|
||||
notFinished = true;
|
||||
|
||||
bool doRealSvd = true;
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsComplex) {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
|
||||
doRealSvd = internal::svd_precondition_2x2_block_to_be_real<MatrixType, Options>::run(m_workMatrix, *this, p,
|
||||
q, maxDiagEntry);
|
||||
}
|
||||
|
||||
@@ -424,10 +424,10 @@ bool SVDBase<Derived>::allocate(Index rows, Index cols, unsigned int computation
|
||||
|
||||
m_diagSize.setValue(numext::mini(m_rows.value(), m_cols.value()));
|
||||
m_singularValues.resize(m_diagSize.value());
|
||||
EIGEN_IF_CONSTEXPR(RowsAtCompileTime == Dynamic) {
|
||||
EIGEN_IF_CONSTEXPR (RowsAtCompileTime == Dynamic) {
|
||||
m_matrixU.resize(m_rows.value(), m_computeFullU ? m_rows.value() : m_computeThinU ? m_diagSize.value() : 0);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(ColsAtCompileTime == Dynamic) {
|
||||
EIGEN_IF_CONSTEXPR (ColsAtCompileTime == Dynamic) {
|
||||
m_matrixV.resize(m_cols.value(), m_computeFullV ? m_cols.value() : m_computeThinV ? m_diagSize.value() : 0);
|
||||
}
|
||||
|
||||
|
||||
@@ -867,7 +867,7 @@ void SimplicialCholeskyBase<Derived>::ordering(const MatrixType& a, ConstCholMat
|
||||
const Index size = a.rows();
|
||||
pmat = ≈
|
||||
// Note that ordering methods compute the inverse permutation
|
||||
EIGEN_IF_CONSTEXPR((!std::is_same<OrderingType, NaturalOrdering<StorageIndex> >::value)) {
|
||||
EIGEN_IF_CONSTEXPR ((!std::is_same<OrderingType, NaturalOrdering<StorageIndex> >::value)) {
|
||||
{
|
||||
CholMatrixType C;
|
||||
constexpr bool kUseAMDFastPath = std::is_same<OrderingType, AMDOrdering<StorageIndex> >::value;
|
||||
@@ -882,16 +882,15 @@ void SimplicialCholeskyBase<Derived>::ordering(const MatrixType& a, ConstCholMat
|
||||
|
||||
ap.resize(size, size);
|
||||
internal::permute_symm_to_symm<UpLo, Upper, NonHermitian>(a, ap, m_P.indices().data());
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_Pinv.resize(0);
|
||||
m_P.resize(0);
|
||||
EIGEN_IF_CONSTEXPR(int(UpLo) == int(Lower) || MatrixType::IsRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (int(UpLo) == int(Lower) || MatrixType::IsRowMajor) {
|
||||
// we have to transpose the lower part to to the upper one
|
||||
ap.resize(size, size);
|
||||
internal::permute_symm_to_symm<UpLo, Upper, NonHermitian>(a, ap, NULL);
|
||||
}
|
||||
else internal::simplicial_cholesky_grab_input<CholMatrixType, MatrixType>::run(a, pmat, ap);
|
||||
} else
|
||||
internal::simplicial_cholesky_grab_input<CholMatrixType, MatrixType>::run(a, pmat, ap);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -376,8 +376,9 @@ void SimplicialCholeskyBase<Derived>::factorize_preordered(const CholMatrixType&
|
||||
Index p = Lp[k] + nonZerosPerCol[k]++;
|
||||
Li[p] = k; /* store L(k,k) = sqrt (d) in column k */
|
||||
bool failed;
|
||||
EIGEN_IF_CONSTEXPR(NonHermitian) { failed = d == RealScalar(0); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (NonHermitian) {
|
||||
failed = d == RealScalar(0);
|
||||
} else {
|
||||
failed = numext::real(d) <= RealScalar(0);
|
||||
}
|
||||
if (failed) {
|
||||
|
||||
@@ -252,9 +252,9 @@ struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Sparse> {
|
||||
template <typename DstXprType, typename SrcXprType, typename Functor, typename Weak>
|
||||
struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Dense, Weak> {
|
||||
static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) {
|
||||
EIGEN_IF_CONSTEXPR(
|
||||
(std::is_same<Functor, internal::assign_op<typename DstXprType::Scalar, typename SrcXprType::Scalar>>::value))
|
||||
dst.setZero();
|
||||
EIGEN_IF_CONSTEXPR ((std::is_same<Functor, internal::assign_op<typename DstXprType::Scalar,
|
||||
typename SrcXprType::Scalar>>::value))
|
||||
dst.setZero();
|
||||
|
||||
internal::evaluator<SrcXprType> srcEval(src);
|
||||
resize_if_allowed(dst, src, func);
|
||||
|
||||
@@ -204,11 +204,10 @@ class sparse_matrix_block_impl : public SparseCompressedBase<Block<SparseMatrixT
|
||||
}
|
||||
|
||||
// update outer index pointers and innerNonZeros
|
||||
EIGEN_IF_CONSTEXPR(IsVectorAtCompileTime) {
|
||||
EIGEN_IF_CONSTEXPR (IsVectorAtCompileTime) {
|
||||
if (!m_matrix.isCompressed()) matrix.innerNonZeroPtr()[m_outerStart] = StorageIndex(nnz);
|
||||
matrix.outerIndexPtr()[m_outerStart] = StorageIndex(start);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
StorageIndex p = StorageIndex(start);
|
||||
for (Index k = 0; k < m_outerSize.value(); ++k) {
|
||||
StorageIndex nnz_k = internal::convert_index<StorageIndex>(tmp.innerVector(k).nonZeros());
|
||||
|
||||
@@ -198,7 +198,7 @@ struct sparse_time_dense_product_impl<SparseLhsType, DenseRhsType, DenseResType,
|
||||
const auto* innerNnz = mat.innerNonZeroPtr();
|
||||
// The fast result pointer path requires contiguous ColMajor result layout.
|
||||
// Transpose<ColMajor> reports innerStride()==1 but is actually RowMajor, so check both.
|
||||
EIGEN_IF_CONSTEXPR(!(Res::Flags & RowMajorBit)) {
|
||||
EIGEN_IF_CONSTEXPR (!(Res::Flags & RowMajorBit)) {
|
||||
if (res.innerStride() == 1) {
|
||||
for (Index c = 0; c < rhs.cols(); ++c) {
|
||||
typename Res::Scalar* y = res.data() + c * res.outerStride();
|
||||
|
||||
@@ -144,8 +144,9 @@ struct sparse_selfadjoint_diagonal_product_impl {
|
||||
const StorageIndex outer = outerIndex(row, col);
|
||||
const Index k = count[outer]++;
|
||||
dest.innerIndexPtr()[k] = innerIndex(row, col);
|
||||
EIGEN_IF_CONSTEXPR(ProductOrder == OnTheLeft) { dest.valuePtr()[k] = diagonal.coeff(row) * coeff; }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (ProductOrder == OnTheLeft) {
|
||||
dest.valuePtr()[k] = diagonal.coeff(row) * coeff;
|
||||
} else {
|
||||
dest.valuePtr()[k] = coeff * diagonal.coeff(col);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,7 +228,7 @@ class SparseMatrixBase : public EigenBase<Derived> {
|
||||
using Nested = typename Derived::Nested;
|
||||
using NestedCleaned = internal::remove_all_t<Nested>;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) {
|
||||
EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) {
|
||||
Nested nm(m.derived());
|
||||
internal::evaluator<NestedCleaned> thisEval(nm);
|
||||
|
||||
@@ -268,8 +268,7 @@ class SparseMatrixBase : public EigenBase<Derived> {
|
||||
}
|
||||
s << std::endl;
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
Nested nm(m.derived());
|
||||
internal::evaluator<NestedCleaned> thisEval(nm);
|
||||
if (m.cols() == 1) {
|
||||
|
||||
@@ -90,18 +90,16 @@ class SparseRefBase : public SparseMapBase<Derived> {
|
||||
protected:
|
||||
template <typename Expression>
|
||||
void construct(Expression& expr) {
|
||||
EIGEN_IF_CONSTEXPR(Expression::IsVectorAtCompileTime) {
|
||||
EIGEN_IF_CONSTEXPR (Expression::IsVectorAtCompileTime) {
|
||||
const Index offset = expr.outerIndexPtr() ? expr.outerIndexPtr()[0] : 0;
|
||||
auto inner_index_ptr = expr.innerIndexPtr();
|
||||
auto value_ptr = expr.valuePtr();
|
||||
if (inner_index_ptr) inner_index_ptr += offset;
|
||||
if (value_ptr) value_ptr += offset;
|
||||
internal::construct_at<Base>(this, expr.size(), expr.nonZeros(), inner_index_ptr, value_ptr);
|
||||
}
|
||||
else if (expr.outerIndexPtr() == 0) {
|
||||
} else if (expr.outerIndexPtr() == 0) {
|
||||
internal::construct_at<Base>(this, expr.size(), expr.nonZeros(), expr.innerIndexPtr(), expr.valuePtr());
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
internal::construct_at<Base>(this, expr.rows(), expr.cols(), expr.nonZeros(), expr.outerIndexPtr(),
|
||||
expr.innerIndexPtr(), expr.valuePtr(), expr.innerNonZeroPtr());
|
||||
}
|
||||
|
||||
@@ -335,7 +335,9 @@ inline void sparse_selfadjoint_time_dense_product(const SparseLhsType& lhs, cons
|
||||
typename DenseResType::Scalar res_j(0);
|
||||
for (; (ProcessFirstHalf ? i && i.index() < j : i); ++i) {
|
||||
LhsScalar lhs_ij = i.value();
|
||||
EIGEN_IF_CONSTEXPR(!LhsIsRowMajor) { lhs_ij = numext::conj(lhs_ij); }
|
||||
EIGEN_IF_CONSTEXPR (!LhsIsRowMajor) {
|
||||
lhs_ij = numext::conj(lhs_ij);
|
||||
}
|
||||
res_j += lhs_ij * rhs.coeff(i.index(), k);
|
||||
res(i.index(), k) += numext::conj(lhs_ij) * rhs_j;
|
||||
}
|
||||
@@ -456,9 +458,10 @@ void permute_symm_to_fullsymm(
|
||||
Index r = it.row();
|
||||
Index c = it.col();
|
||||
Index ip = perm ? perm[i] : i;
|
||||
EIGEN_IF_CONSTEXPR(Mode == int(Upper | Lower))
|
||||
count[StorageOrderMatch ? jp : ip]++;
|
||||
else if (r == c) count[ip]++;
|
||||
EIGEN_IF_CONSTEXPR (Mode == int(Upper | Lower))
|
||||
count[StorageOrderMatch ? jp : ip]++;
|
||||
else if (r == c)
|
||||
count[ip]++;
|
||||
else if ((Mode == Lower && r > c) || (Mode == Upper && r < c)) {
|
||||
count[ip]++;
|
||||
count[jp]++;
|
||||
@@ -483,18 +486,16 @@ void permute_symm_to_fullsymm(
|
||||
StorageIndex jp = perm ? perm[j] : j;
|
||||
StorageIndex ip = perm ? perm[i] : i;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(Mode == int(Upper | Lower)) {
|
||||
EIGEN_IF_CONSTEXPR (Mode == int(Upper | Lower)) {
|
||||
Index k = count[StorageOrderMatch ? jp : ip]++;
|
||||
dest.innerIndexPtr()[k] = StorageOrderMatch ? ip : jp;
|
||||
dest.valuePtr()[k] = it.value();
|
||||
}
|
||||
else if (r == c) {
|
||||
} else if (r == c) {
|
||||
Index k = count[ip]++;
|
||||
dest.innerIndexPtr()[k] = ip;
|
||||
dest.valuePtr()[k] = it.value();
|
||||
}
|
||||
else if (((Mode & Lower) == Lower && r > c) || ((Mode & Upper) == Upper && r < c)) {
|
||||
EIGEN_IF_CONSTEXPR(!StorageOrderMatch) std::swap(ip, jp);
|
||||
} else if (((Mode & Lower) == Lower && r > c) || ((Mode & Upper) == Upper && r < c)) {
|
||||
EIGEN_IF_CONSTEXPR (!StorageOrderMatch) std::swap(ip, jp);
|
||||
Index k = count[jp]++;
|
||||
dest.innerIndexPtr()[k] = ip;
|
||||
dest.valuePtr()[k] = it.value();
|
||||
@@ -556,7 +557,7 @@ void permute_symm_to_symm(const MatrixType& mat,
|
||||
Index k = count[int(DstMode) == int(Lower) ? (std::min)(ip, jp) : (std::max)(ip, jp)]++;
|
||||
dest.innerIndexPtr()[k] = int(DstMode) == int(Lower) ? (std::max)(ip, jp) : (std::min)(ip, jp);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(!StorageOrderMatch) std::swap(ip, jp);
|
||||
EIGEN_IF_CONSTEXPR (!StorageOrderMatch) std::swap(ip, jp);
|
||||
if (((int(DstMode) == int(Lower) && ip < jp) || (int(DstMode) == int(Upper) && ip > jp)))
|
||||
dest.valuePtr()[k] = (NonHermitian ? it.value() : numext::conj(it.value()));
|
||||
else
|
||||
|
||||
@@ -38,8 +38,9 @@ static void sparse_sparse_product_with_pruning_impl(const Lhs& lhs, const Rhs& r
|
||||
AmbiVector<ResScalar, StorageIndex> tempVector(rows);
|
||||
|
||||
// mimics a resizeByInnerOuter:
|
||||
EIGEN_IF_CONSTEXPR(ResultType::IsRowMajor) { res.resize(cols, rows); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (ResultType::IsRowMajor) {
|
||||
res.resize(cols, rows);
|
||||
} else {
|
||||
res.resize(rows, cols);
|
||||
}
|
||||
|
||||
|
||||
@@ -105,12 +105,11 @@ struct unary_evaluator<TriangularView<ArgType, Mode>, IteratorBased> : evaluator
|
||||
: Base(xprEval.m_argImpl, outer),
|
||||
m_returnOne(false),
|
||||
m_containsDiag(Base::outer() < xprEval.m_arg.innerSize()) {
|
||||
EIGEN_IF_CONSTEXPR(SkipFirst) {
|
||||
EIGEN_IF_CONSTEXPR (SkipFirst) {
|
||||
while ((*this) && ((HasUnitDiag || SkipDiag) ? this->index() <= outer : this->index() < outer))
|
||||
Base::operator++();
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) m_returnOne = m_containsDiag;
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(HasUnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) m_returnOne = m_containsDiag;
|
||||
} else EIGEN_IF_CONSTEXPR (HasUnitDiag) {
|
||||
if ((!Base::operator bool()) || Base::index() >= Base::outer()) {
|
||||
if (Base::operator bool()) Base::operator++();
|
||||
m_returnOne = m_containsDiag;
|
||||
@@ -119,14 +118,14 @@ struct unary_evaluator<TriangularView<ArgType, Mode>, IteratorBased> : evaluator
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE InnerIterator& operator++() {
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
|
||||
if (m_returnOne) {
|
||||
m_returnOne = false;
|
||||
return *this;
|
||||
}
|
||||
}
|
||||
Base::operator++();
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag && !SkipFirst) {
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag && !SkipFirst) {
|
||||
if ((!Base::operator bool()) || Base::index() >= Base::outer()) {
|
||||
if (Base::operator bool()) Base::operator++();
|
||||
m_returnOne = m_containsDiag;
|
||||
@@ -136,13 +135,15 @@ struct unary_evaluator<TriangularView<ArgType, Mode>, IteratorBased> : evaluator
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE operator bool() const {
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
|
||||
if (m_returnOne) return true;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(SkipFirst) { return Base::operator bool(); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR(SkipDiag) { return (Base::operator bool() && this->index() < this->outer()); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (SkipFirst) {
|
||||
return Base::operator bool();
|
||||
} else {
|
||||
EIGEN_IF_CONSTEXPR (SkipDiag) {
|
||||
return (Base::operator bool() && this->index() < this->outer());
|
||||
} else {
|
||||
return (Base::operator bool() && this->index() <= this->outer());
|
||||
}
|
||||
}
|
||||
@@ -151,13 +152,13 @@ struct unary_evaluator<TriangularView<ArgType, Mode>, IteratorBased> : evaluator
|
||||
inline Index row() const { return (ArgType::Flags & RowMajorBit ? Base::outer() : this->index()); }
|
||||
inline Index col() const { return (ArgType::Flags & RowMajorBit ? this->index() : Base::outer()); }
|
||||
inline StorageIndex index() const {
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
|
||||
if (m_returnOne) return internal::convert_index<StorageIndex>(Base::outer());
|
||||
}
|
||||
return Base::index();
|
||||
}
|
||||
inline Scalar value() const {
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) {
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
|
||||
if (m_returnOne) return Scalar(1);
|
||||
}
|
||||
return Base::value();
|
||||
|
||||
@@ -382,8 +382,9 @@ void materialize_selfadjoint_pattern(const SparsityPatternRef<StorageIndex>& A,
|
||||
for (Index j = 0; j < n; ++j) {
|
||||
const StorageIndex* a_col = A.inner + A.outer[j];
|
||||
const Index a_nz = A.nonZeros(j);
|
||||
EIGEN_IF_CONSTEXPR(IsLower) { a_split(j) = std::lower_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col; }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (IsLower) {
|
||||
a_split(j) = std::lower_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col;
|
||||
} else {
|
||||
a_split(j) = std::upper_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,8 +44,8 @@ struct sparse_solve_triangular_selector<Lhs, Rhs, Mode, Lower, RowMajor> {
|
||||
if (lastIndex == i) break;
|
||||
tmp = numext::madd<Scalar>(-lastVal, other.coeff(lastIndex, col), tmp);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(Mode & UnitDiag)
|
||||
other.coeffRef(i, col) = tmp;
|
||||
EIGEN_IF_CONSTEXPR (Mode & UnitDiag)
|
||||
other.coeffRef(i, col) = tmp;
|
||||
else {
|
||||
eigen_assert(lastIndex == i);
|
||||
other.coeffRef(i, col) = tmp / lastVal;
|
||||
@@ -69,19 +69,20 @@ struct sparse_solve_triangular_selector<Lhs, Rhs, Mode, Upper, RowMajor> {
|
||||
Scalar l_ii(0);
|
||||
LhsIterator it(lhsEval, i);
|
||||
while (it && it.index() < i) ++it;
|
||||
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) {
|
||||
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) {
|
||||
eigen_assert(it && it.index() == i);
|
||||
l_ii = it.value();
|
||||
++it;
|
||||
}
|
||||
else if (it && it.index() == i)++ it;
|
||||
} else if (it && it.index() == i)
|
||||
++it;
|
||||
for (; it; ++it) {
|
||||
tmp = numext::madd<Scalar>(-it.value(), other.coeff(it.index(), col), tmp);
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(Mode & UnitDiag)
|
||||
other.coeffRef(i, col) = tmp;
|
||||
else other.coeffRef(i, col) = tmp / l_ii;
|
||||
EIGEN_IF_CONSTEXPR (Mode & UnitDiag)
|
||||
other.coeffRef(i, col) = tmp;
|
||||
else
|
||||
other.coeffRef(i, col) = tmp / l_ii;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -102,7 +103,7 @@ struct sparse_solve_triangular_selector<Lhs, Rhs, Mode, Lower, ColMajor> {
|
||||
{
|
||||
LhsIterator it(lhsEval, i);
|
||||
while (it && it.index() < i) ++it;
|
||||
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) {
|
||||
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) {
|
||||
eigen_assert(it && it.index() == i);
|
||||
tmp /= it.value();
|
||||
}
|
||||
@@ -129,7 +130,7 @@ struct sparse_solve_triangular_selector<Lhs, Rhs, Mode, Upper, ColMajor> {
|
||||
Scalar& tmp = other.coeffRef(i, col);
|
||||
if (!numext::is_exactly_zero(tmp)) // optimization when other is actually sparse
|
||||
{
|
||||
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) {
|
||||
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) {
|
||||
// TODO: replace this with a binary search. make sure the binary search is safe for partially sorted
|
||||
// elements
|
||||
LhsIterator it(lhsEval, i);
|
||||
@@ -210,21 +211,20 @@ struct sparse_solve_triangular_sparse_selector<Lhs, Rhs, Mode, UpLo, ColMajor> {
|
||||
if (!numext::is_exactly_zero(ci)) {
|
||||
// find
|
||||
typename Lhs::InnerIterator it(lhs, i);
|
||||
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) {
|
||||
EIGEN_IF_CONSTEXPR(IsLower) {
|
||||
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) {
|
||||
EIGEN_IF_CONSTEXPR (IsLower) {
|
||||
eigen_assert(it.index() == i);
|
||||
ci /= it.value();
|
||||
}
|
||||
else ci /= lhs.coeff(i, i);
|
||||
} else
|
||||
ci /= lhs.coeff(i, i);
|
||||
}
|
||||
tempVector.restart();
|
||||
EIGEN_IF_CONSTEXPR(IsLower) {
|
||||
EIGEN_IF_CONSTEXPR (IsLower) {
|
||||
if (it.index() == i) ++it;
|
||||
for (; it; ++it) {
|
||||
tempVector.coeffRef(it.index()) = numext::madd<Scalar>(-ci, it.value(), tempVector.coeffRef(it.index()));
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (; it && it.index() < i; ++it) {
|
||||
tempVector.coeffRef(it.index()) = numext::madd<Scalar>(-ci, it.value(), tempVector.coeffRef(it.index()));
|
||||
}
|
||||
|
||||
@@ -155,11 +155,14 @@ struct SluMatrix : SuperMatrix {
|
||||
|
||||
template <typename Scalar>
|
||||
void setScalarType() {
|
||||
EIGEN_IF_CONSTEXPR((std::is_same<Scalar, float>::value))
|
||||
Dtype = SLU_S;
|
||||
else EIGEN_IF_CONSTEXPR((std::is_same<Scalar, double>::value)) Dtype = SLU_D;
|
||||
else EIGEN_IF_CONSTEXPR((std::is_same<Scalar, std::complex<float> >::value)) Dtype = SLU_C;
|
||||
else EIGEN_IF_CONSTEXPR((std::is_same<Scalar, std::complex<double> >::value)) Dtype = SLU_Z;
|
||||
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, float>::value))
|
||||
Dtype = SLU_S;
|
||||
else EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, double>::value))
|
||||
Dtype = SLU_D;
|
||||
else EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<float> >::value))
|
||||
Dtype = SLU_C;
|
||||
else EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<double> >::value))
|
||||
Dtype = SLU_Z;
|
||||
else {
|
||||
eigen_assert(false && "Scalar type not supported by SuperLU");
|
||||
}
|
||||
@@ -187,12 +190,11 @@ struct SluMatrix : SuperMatrix {
|
||||
static SluMatrix Map(SparseMatrixBase<MatrixType> &a_mat) {
|
||||
MatrixType &mat(a_mat.derived());
|
||||
SluMatrix res;
|
||||
EIGEN_IF_CONSTEXPR((MatrixType::Flags & RowMajorBit) == RowMajorBit) {
|
||||
EIGEN_IF_CONSTEXPR ((MatrixType::Flags & RowMajorBit) == RowMajorBit) {
|
||||
res.setStorageType(SLU_NR);
|
||||
res.nrow = internal::convert_index<int>(mat.cols());
|
||||
res.ncol = internal::convert_index<int>(mat.rows());
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
res.setStorageType(SLU_NC);
|
||||
res.nrow = internal::convert_index<int>(mat.rows());
|
||||
res.ncol = internal::convert_index<int>(mat.cols());
|
||||
@@ -208,8 +210,8 @@ struct SluMatrix : SuperMatrix {
|
||||
res.setScalarType<typename MatrixType::Scalar>();
|
||||
|
||||
// FIXME: the following type mapping is approximate.
|
||||
EIGEN_IF_CONSTEXPR(int(MatrixType::Flags) & int(Upper)) res.Mtype = SLU_TRU;
|
||||
EIGEN_IF_CONSTEXPR(int(MatrixType::Flags) & int(Lower)) res.Mtype = SLU_TRL;
|
||||
EIGEN_IF_CONSTEXPR (int(MatrixType::Flags) & int(Upper)) res.Mtype = SLU_TRU;
|
||||
EIGEN_IF_CONSTEXPR (int(MatrixType::Flags) & int(Lower)) res.Mtype = SLU_TRL;
|
||||
|
||||
eigen_assert(((int(MatrixType::Flags) & int(SelfAdjoint)) == 0) &&
|
||||
"SelfAdjoint matrix shape not supported by SuperLU");
|
||||
@@ -239,12 +241,11 @@ template <typename Derived>
|
||||
struct SluMatrixMapHelper<SparseMatrixBase<Derived> > {
|
||||
typedef Derived MatrixType;
|
||||
static void run(MatrixType &mat, SluMatrix &res) {
|
||||
EIGEN_IF_CONSTEXPR((MatrixType::Flags & RowMajorBit) == RowMajorBit) {
|
||||
EIGEN_IF_CONSTEXPR ((MatrixType::Flags & RowMajorBit) == RowMajorBit) {
|
||||
res.setStorageType(SLU_NR);
|
||||
res.nrow = mat.cols();
|
||||
res.ncol = mat.rows();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
res.setStorageType(SLU_NC);
|
||||
res.nrow = mat.rows();
|
||||
res.ncol = mat.cols();
|
||||
@@ -260,8 +261,8 @@ struct SluMatrixMapHelper<SparseMatrixBase<Derived> > {
|
||||
res.setScalarType<typename MatrixType::Scalar>();
|
||||
|
||||
// FIXME: the following type mapping is approximate.
|
||||
EIGEN_IF_CONSTEXPR(MatrixType::Flags & Upper) res.Mtype = SLU_TRU;
|
||||
EIGEN_IF_CONSTEXPR(MatrixType::Flags & Lower) res.Mtype = SLU_TRL;
|
||||
EIGEN_IF_CONSTEXPR (MatrixType::Flags & Upper) res.Mtype = SLU_TRU;
|
||||
EIGEN_IF_CONSTEXPR (MatrixType::Flags & Lower) res.Mtype = SLU_TRL;
|
||||
|
||||
eigen_assert(((MatrixType::Flags & SelfAdjoint) == 0) && "SelfAdjoint matrix shape not supported by SuperLU");
|
||||
}
|
||||
|
||||
@@ -197,8 +197,9 @@ class RunQueue {
|
||||
std::atomic_thread_fence(std::memory_order_acquire);
|
||||
continue;
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(NeedSizeEstimate) { return CalculateSize(front, back); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (NeedSizeEstimate) {
|
||||
return CalculateSize(front, back);
|
||||
} else {
|
||||
// This value will be 0 if the queue is empty, and undefined otherwise.
|
||||
unsigned maybe_zero = ((front ^ back) & kMask2);
|
||||
// Queue size estimate must agree with maybe zero check on the queue
|
||||
|
||||
@@ -45,7 +45,7 @@ struct band_solve_triangular_selector<Index, Mode, LhsScalar, ConjLhs, RhsScalar
|
||||
.cwiseProduct(other.col(col).segment(IsLower ? i - actual_k : i + 1, actual_k))
|
||||
.sum();
|
||||
|
||||
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == 0) other.coeffRef(i, col) /= cjLhs(i, IsLower ? k : 0);
|
||||
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) other.coeffRef(i, col) /= cjLhs(i, IsLower ? k : 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -69,7 +69,7 @@ struct band_solve_triangular_selector<Index, Mode, LhsScalar, ConjLhs, RhsScalar
|
||||
int actual_k = (std::min)(k, size - ii - 1);
|
||||
int actual_start = IsLower ? 1 : k - actual_k;
|
||||
|
||||
EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == 0) other.coeffRef(i, col) /= cjLhs(IsLower ? 0 : k, i);
|
||||
EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) other.coeffRef(i, col) /= cjLhs(IsLower ? 0 : k, i);
|
||||
|
||||
if (actual_k > 0)
|
||||
other.col(col).segment(IsLower ? i + 1 : i - actual_k, actual_k) -=
|
||||
|
||||
@@ -38,7 +38,9 @@ struct packed_triangular_matrix_vector_product<Index, Mode, LhsScalar, ConjLhs,
|
||||
if (!(HasUnitDiag || HasZeroDiag) || (--r > 0)) {
|
||||
ResMap(res + (IsLower ? s + i : 0), r) += alpha * cj(rhs[i]) * ConjLhsType(LhsMap(lhs + s, r));
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) { res[i] += alpha * cj(rhs[i]); }
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
|
||||
res[i] += alpha * cj(rhs[i]);
|
||||
}
|
||||
lhs += IsLower ? size - i : i + 1;
|
||||
}
|
||||
};
|
||||
@@ -67,7 +69,9 @@ struct packed_triangular_matrix_vector_product<Index, Mode, LhsScalar, ConjLhs,
|
||||
alpha *
|
||||
(ConjLhsType(LhsMap(lhs + s, r)).cwiseProduct(ConjRhsType(RhsMap(rhs + (IsLower ? 0 : s + i), r)))).sum();
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasUnitDiag) { res[i] += alpha * cj(rhs[i]); }
|
||||
EIGEN_IF_CONSTEXPR (HasUnitDiag) {
|
||||
res[i] += alpha * cj(rhs[i]);
|
||||
}
|
||||
lhs += IsLower ? i + 1 : size - i;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -34,7 +34,7 @@ struct packed_triangular_solve_vector<LhsScalar, RhsScalar, Index, OnTheLeft, Mo
|
||||
rhs[i] -= (ConjLhsType(LhsMap(lhs + s, pi))
|
||||
.cwiseProduct(Map<const Matrix<RhsScalar, Dynamic, 1> >(rhs + (IsLower ? 0 : i + 1), pi)))
|
||||
.sum();
|
||||
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? i : 0]);
|
||||
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? i : 0]);
|
||||
IsLower ? lhs += pi + 1 : lhs -= pi + 2;
|
||||
}
|
||||
}
|
||||
@@ -53,7 +53,7 @@ struct packed_triangular_solve_vector<LhsScalar, RhsScalar, Index, OnTheLeft, Mo
|
||||
for (Index pi = 0; pi < size; ++pi) {
|
||||
Index i = IsLower ? pi : size - pi - 1;
|
||||
Index r = size - pi - 1;
|
||||
EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? 0 : i]);
|
||||
EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? 0 : i]);
|
||||
if (r > 0)
|
||||
Map<Matrix<RhsScalar, Dynamic, 1> >(rhs + (IsLower ? i + 1 : 0), r) -=
|
||||
rhs[i] * ConjLhsType(LhsMap(lhs + (IsLower ? 1 : 0), r));
|
||||
|
||||
@@ -92,12 +92,11 @@ struct unary_evaluator<CoherentPadOp<ArgType, SizeAtCompileTime>>
|
||||
: m_arg(pad.nestedExpression()), m_argImpl(m_arg), m_size(pad.nestedExpression().size()) {}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const {
|
||||
EIGEN_IF_CONSTEXPR(XprType::IsRowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (XprType::IsRowMajor) {
|
||||
if (col < m_size.value()) {
|
||||
return m_argImpl.coeff(1, col);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
if (row < m_size.value()) {
|
||||
return m_argImpl.coeff(row, 1);
|
||||
}
|
||||
@@ -116,7 +115,9 @@ struct unary_evaluator<CoherentPadOp<ArgType, SizeAtCompileTime>>
|
||||
EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
|
||||
// AutoDiff scalar's derivative must be a vector, which is enforced by static assert.
|
||||
// Defer to linear access for simplicity.
|
||||
EIGEN_IF_CONSTEXPR(XprType::IsRowMajor) { return packet(col); }
|
||||
EIGEN_IF_CONSTEXPR (XprType::IsRowMajor) {
|
||||
return packet(col);
|
||||
}
|
||||
return packet(row);
|
||||
}
|
||||
|
||||
|
||||
@@ -94,8 +94,9 @@ template <typename Iterator, bool IsColMajor>
|
||||
struct TripletComp {
|
||||
typedef typename Iterator::value_type Triplet;
|
||||
bool operator()(const Triplet& a, const Triplet& b) {
|
||||
EIGEN_IF_CONSTEXPR(IsColMajor) { return ((a.col() == b.col() && a.row() < b.row()) || (a.col() < b.col())); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (IsColMajor) {
|
||||
return ((a.col() == b.col() && a.row() < b.row()) || (a.col() < b.col()));
|
||||
} else {
|
||||
return ((a.row() == b.row() && a.col() < b.col()) || (a.row() < b.row()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,8 +72,8 @@ inline void GetDenseElt(const std::string& line, std::complex<RealScalar>& val)
|
||||
template <typename Scalar>
|
||||
inline void putMarketHeader(std::string& header, int sym) {
|
||||
header = "%%MatrixMarket matrix coordinate ";
|
||||
EIGEN_IF_CONSTEXPR(
|
||||
(std::is_same<Scalar, std::complex<float> >::value || std::is_same<Scalar, std::complex<double> >::value)) {
|
||||
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<float> >::value ||
|
||||
std::is_same<Scalar, std::complex<double> >::value)) {
|
||||
header += " complex";
|
||||
if (sym == Symmetric)
|
||||
header += " symmetric";
|
||||
@@ -81,8 +81,7 @@ inline void putMarketHeader(std::string& header, int sym) {
|
||||
header += " Hermitian";
|
||||
else
|
||||
header += " general";
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
header += " real";
|
||||
if (sym == Symmetric)
|
||||
header += " symmetric";
|
||||
@@ -340,11 +339,10 @@ bool saveMarketDense(const DenseType& mat, const std::string& filename) {
|
||||
|
||||
out.flags(std::ios_base::scientific);
|
||||
out.precision(std::numeric_limits<RealScalar>::digits10 + 2);
|
||||
EIGEN_IF_CONSTEXPR(
|
||||
(std::is_same<Scalar, std::complex<float> >::value || std::is_same<Scalar, std::complex<double> >::value)) {
|
||||
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<float> >::value ||
|
||||
std::is_same<Scalar, std::complex<double> >::value)) {
|
||||
out << "%%MatrixMarket matrix array complex general\n";
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
out << "%%MatrixMarket matrix array real general\n";
|
||||
}
|
||||
out << mat.rows() << " " << mat.cols() << "\n";
|
||||
|
||||
@@ -179,12 +179,14 @@ class MatrixMarketIterator {
|
||||
if (!getMarketHeader(curfile, m_sym, iscomplex, isvector)) continue;
|
||||
if (isvector) continue;
|
||||
if (!iscomplex) {
|
||||
EIGEN_IF_CONSTEXPR(
|
||||
(std::is_same<Scalar, std::complex<float> >::value || std::is_same<Scalar, std::complex<double> >::value))
|
||||
continue;
|
||||
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, std::complex<float> >::value ||
|
||||
std::is_same<Scalar, std::complex<double> >::value))
|
||||
continue;
|
||||
}
|
||||
if (iscomplex) {
|
||||
EIGEN_IF_CONSTEXPR((std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value)) { continue; }
|
||||
EIGEN_IF_CONSTEXPR ((std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Get the matrix name
|
||||
|
||||
@@ -363,8 +363,9 @@ class Tensor : public TensorBase<Tensor<Scalar_, NumIndices_, Options_, IndexTyp
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index linearizedIndex(const array<Index, NumIndices>& indices) const {
|
||||
EIGEN_IF_CONSTEXPR(Options & RowMajor) { return m_storage.dimensions().IndexOfRowMajor(indices); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Options & RowMajor) {
|
||||
return m_storage.dimensions().IndexOfRowMajor(indices);
|
||||
} else {
|
||||
return m_storage.dimensions().IndexOfColMajor(indices);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,11 +214,10 @@ struct TensorEvaluator<const TensorPairReducerOp<ReduceOp, Dims, ArgType>, Devic
|
||||
m_impl(op.expression().index_pairs().reduce(op.reduce_dims(), op.reduce_op()), device),
|
||||
m_return_dim(op.return_dim()) {
|
||||
gen_strides(m_orig_impl.dimensions(), m_strides);
|
||||
EIGEN_IF_CONSTEXPR(Layout == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == static_cast<int>(ColMajor)) {
|
||||
const Index total_size = internal::array_prod(m_orig_impl.dimensions());
|
||||
m_stride_mod = (m_return_dim < NumDims - 1) ? m_strides[m_return_dim + 1] : total_size;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const Index total_size = internal::array_prod(m_orig_impl.dimensions());
|
||||
m_stride_mod = (m_return_dim > 0) ? m_strides[m_return_dim - 1] : total_size;
|
||||
}
|
||||
@@ -257,13 +256,12 @@ struct TensorEvaluator<const TensorPairReducerOp<ReduceOp, Dims, ArgType>, Devic
|
||||
|
||||
// Calculate m_stride_div and m_stride_mod, which are used to
|
||||
// calculate the value of an index w.r.t. the m_return_dim.
|
||||
EIGEN_IF_CONSTEXPR(Layout == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == static_cast<int>(ColMajor)) {
|
||||
strides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
strides[i] = strides[i - 1] * dims[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
strides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
strides[i] = strides[i + 1] * dims[i + 1];
|
||||
|
||||
@@ -34,13 +34,12 @@ EIGEN_ALWAYS_INLINE DSizes<IndexType, NumDims> strides(const DSizes<IndexType, N
|
||||
|
||||
// TODO(ezhulenev): Use templates to unroll this loop (similar to
|
||||
// h_array_reduce in MoreMeta.h)? Benchmark it.
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
strides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
strides[i] = strides[i - 1] * dimensions[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
strides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
strides[i] = strides[i + 1] * dimensions[i + 1];
|
||||
@@ -952,7 +951,7 @@ class StridedLinearBufferCopy {
|
||||
const Scalar* src = &src_data[src_offset];
|
||||
Scalar* dst = &dst_data[dst_offset];
|
||||
|
||||
EIGEN_IF_CONSTEXPR(!Vectorizable) {
|
||||
EIGEN_IF_CONSTEXPR (!Vectorizable) {
|
||||
for (Index i = 0; i < count; ++i) {
|
||||
dst[i * dst_stride] = src[i * src_stride];
|
||||
}
|
||||
@@ -962,7 +961,7 @@ class StridedLinearBufferCopy {
|
||||
const IndexType vectorized_size = PacketSize * (count / PacketSize);
|
||||
IndexType i = 0;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Linear) {
|
||||
EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Linear) {
|
||||
// ******************************************************************** //
|
||||
// Linear copy from `src` to `dst`.
|
||||
const IndexType unrolled_size = (4 * PacketSize) * (count / (4 * PacketSize));
|
||||
@@ -977,7 +976,7 @@ class StridedLinearBufferCopy {
|
||||
Packet p = ploadu<Packet>(src + i);
|
||||
pstoreu<Scalar, Packet>(dst + i, p);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
|
||||
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
|
||||
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
|
||||
if (i < vectorized_half_size) {
|
||||
HalfPacket p = ploadu<HalfPacket>(src + i);
|
||||
@@ -989,15 +988,14 @@ class StridedLinearBufferCopy {
|
||||
dst[i] = src[i];
|
||||
}
|
||||
// ******************************************************************** //
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Scatter) {
|
||||
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Scatter) {
|
||||
// Scatter from `src` to `dst`.
|
||||
eigen_assert(src_stride == 1 && dst_stride != 1);
|
||||
for (; i < vectorized_size; i += PacketSize) {
|
||||
Packet p = ploadu<Packet>(src + i);
|
||||
pscatter<Scalar, Packet>(dst + i * dst_stride, p, dst_stride);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
|
||||
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
|
||||
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
|
||||
if (i < vectorized_half_size) {
|
||||
HalfPacket p = ploadu<HalfPacket>(src + i);
|
||||
@@ -1009,8 +1007,7 @@ class StridedLinearBufferCopy {
|
||||
dst[i * dst_stride] = src[i];
|
||||
}
|
||||
// ******************************************************************** //
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::FillLinear) {
|
||||
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::FillLinear) {
|
||||
// Fill `dst` with value at `*src`.
|
||||
eigen_assert(src_stride == 0 && dst_stride == 1);
|
||||
|
||||
@@ -1025,7 +1022,7 @@ class StridedLinearBufferCopy {
|
||||
for (; i < vectorized_size; i += PacketSize) {
|
||||
pstoreu<Scalar, Packet>(dst + i, p);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
|
||||
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
|
||||
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
|
||||
if (i < vectorized_half_size) {
|
||||
HalfPacket hp = pset1<HalfPacket>(s);
|
||||
@@ -1037,8 +1034,7 @@ class StridedLinearBufferCopy {
|
||||
dst[i] = s;
|
||||
}
|
||||
// ******************************************************************** //
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::FillScatter) {
|
||||
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::FillScatter) {
|
||||
// Scatter `*src` into `dst`.
|
||||
eigen_assert(src_stride == 0 && dst_stride != 1);
|
||||
Scalar s = *src;
|
||||
@@ -1046,7 +1042,7 @@ class StridedLinearBufferCopy {
|
||||
for (; i < vectorized_size; i += PacketSize) {
|
||||
pscatter<Scalar, Packet>(dst + i * dst_stride, p, dst_stride);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
|
||||
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
|
||||
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
|
||||
if (i < vectorized_half_size) {
|
||||
HalfPacket hp = pset1<HalfPacket>(s);
|
||||
@@ -1058,15 +1054,14 @@ class StridedLinearBufferCopy {
|
||||
dst[i * dst_stride] = s;
|
||||
}
|
||||
// ******************************************************************** //
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Gather) {
|
||||
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Gather) {
|
||||
// Gather from `src` into `dst`.
|
||||
eigen_assert(dst_stride == 1);
|
||||
for (; i < vectorized_size; i += PacketSize) {
|
||||
Packet p = pgather<Scalar, Packet>(src + i * src_stride, src_stride);
|
||||
pstoreu<Scalar, Packet>(dst + i, p);
|
||||
}
|
||||
EIGEN_IF_CONSTEXPR(HasHalfPacket) {
|
||||
EIGEN_IF_CONSTEXPR (HasHalfPacket) {
|
||||
const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
|
||||
if (i < vectorized_half_size) {
|
||||
HalfPacket p = pgather<Scalar, HalfPacket>(src + i * src_stride, src_stride);
|
||||
@@ -1078,14 +1073,12 @@ class StridedLinearBufferCopy {
|
||||
dst[i] = src[i * src_stride];
|
||||
}
|
||||
// ******************************************************************** //
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Random) {
|
||||
} else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Random) {
|
||||
// Random.
|
||||
for (; i < count; ++i) {
|
||||
dst[i * dst_stride] = src[i * src_stride];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
eigen_assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,15 +145,14 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_inputStrides[0] = 1;
|
||||
m_outputStrides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1];
|
||||
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_inputStrides[NumDims - 1] = 1;
|
||||
m_outputStrides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
@@ -218,14 +217,13 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
|
||||
return m_impl.coeff(0);
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
if (isCopy) {
|
||||
return m_impl.coeff(index);
|
||||
} else {
|
||||
return coeffColMajor(index);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
if (isCopy) {
|
||||
return m_impl.coeff(index);
|
||||
} else {
|
||||
@@ -314,7 +312,7 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
|
||||
return internal::pset1<PacketReturnType>(m_impl.coeff(0));
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
if (isCopy) {
|
||||
#ifdef EIGEN_GPU_COMPILE_PHASE
|
||||
// See PR 437: on NVIDIA P100 and K20m we observed a x3-4 speed up by enforcing
|
||||
@@ -332,8 +330,7 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
|
||||
} else {
|
||||
return packetColMajor<LoadMode>(index);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
if (isCopy) {
|
||||
#ifdef EIGEN_GPU_COMPILE_PHASE
|
||||
// See above.
|
||||
@@ -361,11 +358,10 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> {
|
||||
Index startDim, endDim;
|
||||
Index inputIndex, outputOffset, batchedIndex;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
startDim = NumDims - 1;
|
||||
endDim = 1;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
startDim = 0;
|
||||
endDim = NumDims - 2;
|
||||
}
|
||||
|
||||
@@ -156,13 +156,12 @@ struct TensorEvaluator<const TensorChippingOp<DimId, ArgType>, Device> {
|
||||
|
||||
m_stride = 1;
|
||||
m_inputStride = 1;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = 0; i < m_dim.actualDim(); ++i) {
|
||||
m_stride *= input_dims[i];
|
||||
m_inputStride *= input_dims[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = NumInputDims - 1; i > m_dim.actualDim(); --i) {
|
||||
m_stride *= input_dims[i];
|
||||
m_inputStride *= input_dims[i];
|
||||
@@ -183,11 +182,10 @@ struct TensorEvaluator<const TensorChippingOp<DimId, ArgType>, Device> {
|
||||
before_chipped_dim_product *= input_dims[i];
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_isEffectivelyInnerChipping = before_chipped_dim_product == 1;
|
||||
m_isEffectivelyOuterChipping = after_chipped_dim_product == 1;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_isEffectivelyInnerChipping = after_chipped_dim_product == 1;
|
||||
m_isEffectivelyOuterChipping = before_chipped_dim_product == 1;
|
||||
}
|
||||
|
||||
@@ -164,7 +164,7 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_leftStrides[0] = 1;
|
||||
m_rightStrides[0] = 1;
|
||||
m_outputStrides[0] = 1;
|
||||
@@ -174,8 +174,7 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
|
||||
m_rightStrides[j] = m_rightStrides[j - 1] * rhs_dims[j - 1];
|
||||
m_outputStrides[j] = m_outputStrides[j - 1] * m_dimensions[j - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_leftStrides[NumDims - 1] = 1;
|
||||
m_rightStrides[NumDims - 1] = 1;
|
||||
m_outputStrides[NumDims - 1] = 1;
|
||||
@@ -361,14 +360,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
|
||||
// Collect dimension-wise indices (subs).
|
||||
array<Index, NumDims> subs;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
subs[i] = index / m_outputStrides[i];
|
||||
index -= subs[i] * m_outputStrides[i];
|
||||
}
|
||||
subs[0] = index;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
subs[i] = index / m_outputStrides[i];
|
||||
index -= subs[i] * m_outputStrides[i];
|
||||
@@ -379,14 +377,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
|
||||
const Dimensions& left_dims = m_leftImpl.dimensions();
|
||||
if (subs[m_axis] < left_dims[m_axis]) {
|
||||
Index left_index;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
left_index = subs[0];
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
left_index += (subs[i] % left_dims[i]) * m_leftStrides[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
left_index = subs[NumDims - 1];
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
@@ -398,14 +395,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
|
||||
subs[m_axis] -= left_dims[m_axis];
|
||||
const Dimensions& right_dims = m_rightImpl.dimensions();
|
||||
Index right_index;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
right_index = subs[0];
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
right_index += (subs[i] % right_dims[i]) * m_rightStrides[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
right_index = subs[NumDims - 1];
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
@@ -436,7 +432,7 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
|
||||
array<Index, NumDims> subs_end;
|
||||
Index remaining = index;
|
||||
Index remaining_end = index + packetSize - 1;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
subs[i] = remaining / m_outputStrides[i];
|
||||
remaining -= subs[i] * m_outputStrides[i];
|
||||
@@ -445,8 +441,7 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
|
||||
}
|
||||
subs[0] = remaining;
|
||||
subs_end[0] = remaining_end;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
subs[i] = remaining / m_outputStrides[i];
|
||||
remaining -= subs[i] * m_outputStrides[i];
|
||||
@@ -476,14 +471,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
|
||||
|
||||
if (on_left) {
|
||||
Index left_index;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
left_index = subs[0];
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
left_index += subs[i] * m_leftStrides[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
left_index = subs[NumDims - 1];
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
@@ -495,14 +489,13 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy
|
||||
if (on_right) {
|
||||
subs[m_axis] -= left_axis_size;
|
||||
Index right_index;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
right_index = subs[0];
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
right_index += subs[i] * m_rightStrides[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
right_index = subs[NumDims - 1];
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
|
||||
@@ -466,7 +466,7 @@ struct TensorContractionEvaluatorBase {
|
||||
DSizes<Index, LDims> eval_left_dims;
|
||||
DSizes<Index, RDims> eval_right_dims;
|
||||
array<IndexPair<Index>, ContractDims> eval_op_indices;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
// For ColMajor, we keep using the existing dimensions
|
||||
for (int i = 0; i < LDims; i++) {
|
||||
eval_left_dims[i] = m_leftImpl.dimensions()[i];
|
||||
@@ -479,8 +479,7 @@ struct TensorContractionEvaluatorBase {
|
||||
eval_op_indices[i].first = op.indices()[i].first;
|
||||
eval_op_indices[i].second = op.indices()[i].second;
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// For RowMajor, we need to reverse the existing dimensions
|
||||
for (int i = 0; i < LDims; i++) {
|
||||
eval_left_dims[i] = m_leftImpl.dimensions()[LDims - i - 1];
|
||||
@@ -642,7 +641,7 @@ struct TensorContractionEvaluatorBase {
|
||||
}
|
||||
|
||||
// If the layout is RowMajor, we need to reverse the m_dimensions
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(RowMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(RowMajor)) {
|
||||
for (int i = 0, j = NumDims - 1; i < j; i++, j--) {
|
||||
numext::swap(m_dimensions[i], m_dimensions[j]);
|
||||
}
|
||||
|
||||
@@ -265,13 +265,12 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT
|
||||
|
||||
internal::tensor_contraction_dispatch(
|
||||
[&](auto lhs_c, auto rhs_c, auto rhs_r) {
|
||||
EIGEN_IF_CONSTEXPR(IsEvalInSyncMode) {
|
||||
EIGEN_IF_CONSTEXPR (IsEvalInSyncMode) {
|
||||
EvalParallelContext<NoCallback, lhs_c(), rhs_c(), rhs_r(), Alignment> ctx(
|
||||
this, num_threads, buffer, m, n, k, bm, bn, bk, nm, nn, nk, gm, gn, nm0, nn0, shard_by_col,
|
||||
parallel_pack, parallelize_by_sharding_dim_only, NoCallback());
|
||||
ctx.run();
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
auto* ctx = new EvalParallelContext<DoneCallback, lhs_c(), rhs_c(), rhs_r(), Alignment>(
|
||||
this, num_threads, buffer, m, n, k, bm, bn, bk, nm, nn, nk, gm, gn, nm0, nn0, shard_by_col,
|
||||
parallel_pack, parallelize_by_sharding_dim_only, std::move(done));
|
||||
|
||||
@@ -34,15 +34,14 @@ class IndexMapper {
|
||||
|
||||
array<Index, NumDims> inputStrides;
|
||||
array<Index, NumDims> outputStrides;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
inputStrides[0] = 1;
|
||||
outputStrides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
inputStrides[i] = inputStrides[i - 1] * input_dims[i - 1];
|
||||
outputStrides[i] = outputStrides[i - 1] * dimensions[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
inputStrides[NumDims - 1] = 1;
|
||||
outputStrides[NumDims - 1] = 1;
|
||||
for (int i = static_cast<int>(NumDims) - 2; i >= 0; --i) {
|
||||
@@ -79,7 +78,7 @@ class IndexMapper {
|
||||
m_outputStrides[i] = outputStrides[ordering[i]];
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = 0; i < NumDims; ++i) {
|
||||
if (i > NumKernelDims) {
|
||||
m_gpuInputStrides[i] = m_gpuInputStrides[i - 1] * gpuInputDimensions[i - 1];
|
||||
@@ -89,8 +88,7 @@ class IndexMapper {
|
||||
m_gpuOutputStrides[i] = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = NumDims - 1; i >= 0; --i) {
|
||||
if (i + 1 < static_cast<int>(offset)) {
|
||||
m_gpuInputStrides[i] = m_gpuInputStrides[i + 1] * gpuInputDimensions[i + 1];
|
||||
@@ -105,7 +103,7 @@ class IndexMapper {
|
||||
|
||||
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputPlaneToTensorInputOffset(Index p) const {
|
||||
Index inputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int d = NumDims - 1; d > NumKernelDims; --d) {
|
||||
const Index idx = p / m_gpuInputStrides[d];
|
||||
inputIndex += idx * m_inputStrides[d];
|
||||
@@ -114,8 +112,7 @@ class IndexMapper {
|
||||
if (NumKernelDims < NumDims) {
|
||||
inputIndex += p * m_inputStrides[NumKernelDims];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
std::ptrdiff_t limit = 0;
|
||||
if (NumKernelDims < NumDims) {
|
||||
limit = NumDims - NumKernelDims - 1;
|
||||
@@ -132,7 +129,7 @@ class IndexMapper {
|
||||
|
||||
EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputPlaneToTensorOutputOffset(Index p) const {
|
||||
Index outputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int d = NumDims - 1; d > NumKernelDims; --d) {
|
||||
const Index idx = p / m_gpuOutputStrides[d];
|
||||
outputIndex += idx * m_outputStrides[d];
|
||||
@@ -141,8 +138,7 @@ class IndexMapper {
|
||||
if (NumKernelDims < NumDims) {
|
||||
outputIndex += p * m_outputStrides[NumKernelDims];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
std::ptrdiff_t limit = 0;
|
||||
if (NumKernelDims < NumDims) {
|
||||
limit = NumDims - NumKernelDims - 1;
|
||||
@@ -314,13 +310,12 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
|
||||
const typename TensorEvaluator<InputArgType, Device>::Dimensions& input_dims = m_inputImpl.dimensions();
|
||||
const typename TensorEvaluator<KernelArgType, Device>::Dimensions& kernel_dims = m_kernelImpl.dimensions();
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_inputStride[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_inputStride[i] = m_inputStride[i - 1] * input_dims[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_inputStride[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
m_inputStride[i] = m_inputStride[i + 1] * input_dims[i + 1];
|
||||
@@ -328,7 +323,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
|
||||
}
|
||||
|
||||
m_dimensions = m_inputImpl.dimensions();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = 0; i < NumKernelDims; ++i) {
|
||||
const Index index = op.indices()[i];
|
||||
const Index input_dim = input_dims[index];
|
||||
@@ -347,8 +342,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_outputStride[i] = m_outputStride[i - 1] * m_dimensions[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = NumKernelDims - 1; i >= 0; --i) {
|
||||
const Index index = op.indices()[i];
|
||||
const Index input_dim = input_dims[index];
|
||||
@@ -404,7 +398,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
|
||||
EIGEN_DEVICE_FUNC PacketReturnType packet(const Index index) const {
|
||||
Index indices[2] = {index, index + PacketSize - 1};
|
||||
Index startInputs[2] = {0, 0};
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx0 = indices[0] / m_outputStride[i];
|
||||
const Index idx1 = indices[1] / m_outputStride[i];
|
||||
@@ -413,8 +407,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
|
||||
indices[0] -= idx0 * m_outputStride[i];
|
||||
indices[1] -= idx1 * m_outputStride[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx0 = indices[0] / m_outputStride[i];
|
||||
const Index idx1 = indices[1] / m_outputStride[i];
|
||||
@@ -462,14 +455,13 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr
|
||||
private:
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const {
|
||||
Index startInput = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = index / m_outputStride[i];
|
||||
startInput += idx * m_inputStride[i];
|
||||
index -= idx * m_outputStride[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = index / m_outputStride[i];
|
||||
startInput += idx * m_inputStride[i];
|
||||
|
||||
@@ -120,20 +120,18 @@ struct TensorEvaluator {
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<DenseIndex, NumCoords>& coords) const {
|
||||
eigen_assert(m_data != NULL);
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
return m_data[m_dims.IndexOfColMajor(coords)];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return m_data[m_dims.IndexOfRowMajor(coords)];
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(const array<DenseIndex, NumCoords>& coords) const {
|
||||
eigen_assert(m_data != NULL);
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
return m_data[m_dims.IndexOfColMajor(coords)];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return m_data[m_dims.IndexOfRowMajor(coords)];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,13 +179,12 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D
|
||||
m_dimensions[i] = input_dims[i];
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_strides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_strides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1];
|
||||
@@ -644,7 +643,7 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index getBaseOffsetFromIndex(Index index, Index omitted_dim) const {
|
||||
Index result = 0;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > omitted_dim; --i) {
|
||||
const Index partial_m_stride = m_strides[i] / m_dimensions[omitted_dim];
|
||||
const Index idx = index / partial_m_stride;
|
||||
@@ -652,8 +651,7 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D
|
||||
result += idx * m_strides[i];
|
||||
}
|
||||
result += index;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (Index i = 0; i < omitted_dim; ++i) {
|
||||
const Index partial_m_stride = m_strides[i] / m_dimensions[omitted_dim];
|
||||
const Index idx = index / partial_m_stride;
|
||||
|
||||
@@ -203,8 +203,9 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_,
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index linearizedIndex(const array<Index, NumIndices>& indices) const {
|
||||
EIGEN_IF_CONSTEXPR(Options & RowMajor) { return m_storage.dimensions().IndexOfRowMajor(indices); }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (Options & RowMajor) {
|
||||
return m_storage.dimensions().IndexOfRowMajor(indices);
|
||||
} else {
|
||||
return m_storage.dimensions().IndexOfColMajor(indices);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,15 +105,14 @@ struct TensorEvaluator<const TensorGeneratorOp<Generator, ArgType>, Device> {
|
||||
TensorEvaluator<ArgType, Device> argImpl(op.expression(), device);
|
||||
m_dimensions = argImpl.dimensions();
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_strides[0] = 1;
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1];
|
||||
if (m_strides[i] != 0) m_fast_strides[i] = IndexDivisor(m_strides[i]);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_strides[NumDims - 1] = 1;
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
@@ -242,15 +241,14 @@ struct TensorEvaluator<const TensorGeneratorOp<Generator, ArgType>, Device> {
|
||||
|
||||
protected:
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void extract_coordinates(Index index, array<Index, NumDims>& coords) const {
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = index / m_fast_strides[i];
|
||||
index -= idx * m_strides[i];
|
||||
coords[i] = idx;
|
||||
}
|
||||
coords[0] = index;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = index / m_fast_strides[i];
|
||||
index -= idx * m_strides[i];
|
||||
|
||||
@@ -238,8 +238,9 @@ struct TensorPrinter {
|
||||
if (fmt.precision == StreamPrecision) {
|
||||
explicit_precision = 0;
|
||||
} else if (fmt.precision == FullPrecision) {
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsInteger) { explicit_precision = 0; }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsInteger) {
|
||||
explicit_precision = 0;
|
||||
} else {
|
||||
explicit_precision = significant_decimals_impl<Scalar>::run();
|
||||
}
|
||||
} else {
|
||||
@@ -385,8 +386,9 @@ struct TensorPrinter<Tensor, 0, Format> {
|
||||
if (fmt.precision == StreamPrecision) {
|
||||
explicit_precision = 0;
|
||||
} else if (fmt.precision == FullPrecision) {
|
||||
EIGEN_IF_CONSTEXPR(NumTraits<Scalar>::IsInteger) { explicit_precision = 0; }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsInteger) {
|
||||
explicit_precision = 0;
|
||||
} else {
|
||||
explicit_precision = significant_decimals_impl<Scalar>::run();
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -193,12 +193,11 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
|
||||
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
|
||||
|
||||
// Caches a few variables.
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_inputDepth = input_dims[0];
|
||||
m_inputRows = input_dims[1];
|
||||
m_inputCols = input_dims[2];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_inputDepth = input_dims[NumInputDims - 1];
|
||||
m_inputRows = input_dims[NumInputDims - 2];
|
||||
m_inputCols = input_dims[NumInputDims - 3];
|
||||
@@ -270,7 +269,7 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
|
||||
eigen_assert(m_outputCols > 0);
|
||||
|
||||
// Dimensions for result of extraction.
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
// ColMajor
|
||||
// 0: depth
|
||||
// 1: patch_rows
|
||||
@@ -284,8 +283,7 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
|
||||
for (int i = 4; i < NumDims; ++i) {
|
||||
m_dimensions[i] = input_dims[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// RowMajor
|
||||
// NumDims-1: depth
|
||||
// NumDims-2: patch_rows
|
||||
@@ -302,12 +300,11 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
|
||||
}
|
||||
|
||||
// Strides for moving the patch in various dimensions.
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_colStride = m_dimensions[1];
|
||||
m_patchStride = m_colStride * m_dimensions[2] * m_dimensions[0];
|
||||
m_otherStride = m_patchStride * m_dimensions[3];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_colStride = m_dimensions[NumDims - 2];
|
||||
m_patchStride = m_colStride * m_dimensions[NumDims - 3] * m_dimensions[NumDims - 1];
|
||||
m_otherStride = m_patchStride * m_dimensions[NumDims - 4];
|
||||
@@ -328,10 +325,9 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> {
|
||||
|
||||
// Number of patches in the width dimension.
|
||||
m_fastOutputRows = internal::TensorIntDivisor<Index>(m_outputRows);
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[0]);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[NumDims - 1]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,15 +111,14 @@ struct TensorEvaluator<const TensorInflationOp<Strides, ArgType>, Device> {
|
||||
}
|
||||
|
||||
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_outputStrides[0] = 1;
|
||||
m_inputStrides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
|
||||
m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1];
|
||||
}
|
||||
}
|
||||
else { // RowMajor
|
||||
} else { // RowMajor
|
||||
m_outputStrides[NumDims - 1] = 1;
|
||||
m_inputStrides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
@@ -142,7 +141,7 @@ struct TensorEvaluator<const TensorInflationOp<Strides, ArgType>, Device> {
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool getInputIndex(Index index, Index* inputIndex) const {
|
||||
eigen_assert(index < dimensions().TotalSize());
|
||||
*inputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = index / m_outputStrides[i];
|
||||
@@ -157,8 +156,7 @@ struct TensorEvaluator<const TensorInflationOp<Strides, ArgType>, Device> {
|
||||
}
|
||||
*inputIndex += index / m_strides[0];
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = index / m_outputStrides[i];
|
||||
|
||||
@@ -105,11 +105,10 @@ class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_, MakePoi
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StoragePointerType data() const { return m_data; }
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageRefType operator()(const array<Index, NumIndices>& indices) const {
|
||||
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
|
||||
const Index index = m_dimensions.IndexOfRowMajor(indices);
|
||||
return m_data[index];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const Index index = m_dimensions.IndexOfColMajor(indices);
|
||||
return m_data[index];
|
||||
}
|
||||
@@ -130,12 +129,11 @@ class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_, MakePoi
|
||||
IndexTypes... otherIndices) const {
|
||||
EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 2 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE)
|
||||
eigen_assert(internal::all((Eigen::NumTraits<Index>::highest() >= otherIndices)...));
|
||||
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
|
||||
const Index index =
|
||||
m_dimensions.IndexOfRowMajor(array<Index, NumIndices>{{firstIndex, secondIndex, otherIndices...}});
|
||||
return m_data[index];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const Index index =
|
||||
m_dimensions.IndexOfColMajor(array<Index, NumIndices>{{firstIndex, secondIndex, otherIndices...}});
|
||||
return m_data[index];
|
||||
@@ -143,11 +141,10 @@ class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_, MakePoi
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE StorageRefType operator()(const array<Index, NumIndices>& indices) {
|
||||
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
|
||||
const Index index = m_dimensions.IndexOfRowMajor(indices);
|
||||
return m_data[index];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const Index index = m_dimensions.IndexOfColMajor(indices);
|
||||
return m_data[index];
|
||||
}
|
||||
@@ -170,12 +167,11 @@ class TensorMap : public TensorBase<TensorMap<PlainObjectType, Options_, MakePoi
|
||||
"Number of indices used to access a tensor coefficient must be equal to the rank of the tensor.");
|
||||
eigen_assert(internal::all((Eigen::NumTraits<Index>::highest() >= otherIndices)...));
|
||||
const std::size_t NumDims = sizeof...(otherIndices) + 2;
|
||||
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
|
||||
const Index index =
|
||||
m_dimensions.IndexOfRowMajor(array<Index, NumDims>{{firstIndex, secondIndex, otherIndices...}});
|
||||
return m_data[index];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
const Index index =
|
||||
m_dimensions.IndexOfColMajor(array<Index, NumDims>{{firstIndex, secondIndex, otherIndices...}});
|
||||
return m_data[index];
|
||||
|
||||
@@ -399,7 +399,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
|
||||
|
||||
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
|
||||
const Sizes& output_dims = op.sizes();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_inputStrides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1];
|
||||
@@ -411,8 +411,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
|
||||
m_outputStrides[i] = m_outputStrides[i - 1] * output_dims[i - 1];
|
||||
m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_inputStrides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
m_inputStrides[i] = m_inputStrides[i + 1] * input_dims[i + 1];
|
||||
@@ -433,15 +432,14 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
|
||||
m_impl.evalSubExprsIfNeeded(NULL);
|
||||
if (!NumTraits<std::remove_const_t<Scalar>>::RequireInitialization && data && m_impl.data()) {
|
||||
Index contiguous_values = 1;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = 0; i < NumDims; ++i) {
|
||||
contiguous_values *= dimensions()[i];
|
||||
if (dimensions()[i] != m_impl.dimensions()[i]) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = NumDims - 1; i >= 0; --i) {
|
||||
contiguous_values *= dimensions()[i];
|
||||
if (dimensions()[i] != m_impl.dimensions()[i]) {
|
||||
@@ -493,7 +491,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
|
||||
|
||||
Index inputIndices[] = {0, 0};
|
||||
Index indices[] = {index, index + packetSize - 1};
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx0 = indices[0] / m_fastOutputStrides[i];
|
||||
@@ -505,8 +503,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
|
||||
}
|
||||
inputIndices[0] += (indices[0] + m_offsets[0]);
|
||||
inputIndices[1] += (indices[1] + m_offsets[0]);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx0 = indices[0] / m_fastOutputStrides[i];
|
||||
@@ -557,7 +554,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
|
||||
typename Storage::Type result = constCast(m_impl.data());
|
||||
if (result) {
|
||||
Index offset = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = 0; i < NumDims; ++i) {
|
||||
if (m_dimensions[i] != m_impl.dimensions()[i]) {
|
||||
offset += m_offsets[i] * m_inputStrides[i];
|
||||
@@ -570,8 +567,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = NumDims - 1; i >= 0; --i) {
|
||||
if (m_dimensions[i] != m_impl.dimensions()[i]) {
|
||||
offset += m_offsets[i] * m_inputStrides[i];
|
||||
@@ -593,7 +589,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
|
||||
protected:
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
|
||||
Index inputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = index / m_fastOutputStrides[i];
|
||||
@@ -601,8 +597,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi
|
||||
index -= idx * m_outputStrides[i];
|
||||
}
|
||||
inputIndex += (index + m_offsets[0]);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = index / m_fastOutputStrides[i];
|
||||
@@ -675,7 +670,7 @@ struct TensorEvaluator<TensorSlicingOp<StartIndices, Sizes, ArgType>, Device>
|
||||
const int packetSize = PacketType<CoeffReturnType, Device>::size;
|
||||
Index inputIndices[] = {0, 0};
|
||||
Index indices[] = {index, index + packetSize - 1};
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx0 = indices[0] / this->m_fastOutputStrides[i];
|
||||
@@ -687,8 +682,7 @@ struct TensorEvaluator<TensorSlicingOp<StartIndices, Sizes, ArgType>, Device>
|
||||
}
|
||||
inputIndices[0] += (indices[0] + this->m_offsets[0]);
|
||||
inputIndices[1] += (indices[1] + this->m_offsets[0]);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx0 = indices[0] / this->m_fastOutputStrides[i];
|
||||
@@ -844,7 +838,7 @@ struct TensorEvaluator<const TensorStridingSlicingOp<StartIndices, StopIndices,
|
||||
|
||||
Strides output_dims = m_dimensions;
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_inputStrides[0] = m_strides[0];
|
||||
m_offsets[0] = startIndicesClamped[0];
|
||||
Index previousDimProduct = 1;
|
||||
@@ -860,8 +854,7 @@ struct TensorEvaluator<const TensorStridingSlicingOp<StartIndices, StopIndices,
|
||||
m_outputStrides[i] = m_outputStrides[i - 1] * output_dims[i - 1];
|
||||
m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_inputStrides[NumDims - 1] = m_strides[NumDims - 1];
|
||||
m_offsets[NumDims - 1] = startIndicesClamped[NumDims - 1];
|
||||
Index previousDimProduct = 1;
|
||||
@@ -905,15 +898,14 @@ struct TensorEvaluator<const TensorStridingSlicingOp<StartIndices, StopIndices,
|
||||
protected:
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
|
||||
Index inputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i >= 0; --i) {
|
||||
const Index idx = index / m_fastOutputStrides[i];
|
||||
inputIndex += idx * m_inputStrides[i] + m_offsets[i];
|
||||
index -= idx * m_outputStrides[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims; ++i) {
|
||||
const Index idx = index / m_fastOutputStrides[i];
|
||||
|
||||
@@ -121,7 +121,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
|
||||
m_dimensions[i] += m_padding[i].first + m_padding[i].second;
|
||||
}
|
||||
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_inputStrides[0] = 1;
|
||||
m_outputStrides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
@@ -129,8 +129,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
|
||||
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
|
||||
}
|
||||
m_outputStrides[NumDims] = m_outputStrides[NumDims - 1] * m_dimensions[NumDims - 1];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_inputStrides[NumDims - 1] = 1;
|
||||
m_outputStrides[NumDims] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
@@ -160,7 +159,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
|
||||
eigen_assert(index < dimensions().TotalSize());
|
||||
Index inputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = index / m_outputStrides[i];
|
||||
@@ -174,8 +173,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
|
||||
return m_paddingValue;
|
||||
}
|
||||
inputIndex += (index - m_padding[0].first);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = index / m_outputStrides[i + 1];
|
||||
@@ -195,17 +193,18 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device
|
||||
|
||||
template <int LoadMode>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) { return packetColMajor(index); }
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
return packetColMajor(index);
|
||||
}
|
||||
return packetRowMajor(index);
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
|
||||
TensorOpCost cost = m_impl.costPerCoeff(vectorized);
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims; ++i) updateCostPerDimension(cost, i, i == 0);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i >= 0; --i) updateCostPerDimension(cost, i, i == NumDims - 1);
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
|
||||
Index num_patches = 1;
|
||||
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
|
||||
const PatchDim& patch_dims = op.patch_dims();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
m_dimensions[i] = patch_dims[i];
|
||||
num_patches *= (input_dims[i] - patch_dims[i] + 1);
|
||||
@@ -118,8 +118,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
m_dimensions[i + 1] = patch_dims[i];
|
||||
num_patches *= (input_dims[i] - patch_dims[i] + 1);
|
||||
@@ -155,7 +154,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
|
||||
// Find the offset of the element wrt the location of the first element.
|
||||
Index patchOffset = index - patchIndex * m_outputStrides[output_stride_index];
|
||||
Index inputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 2; i > 0; --i) {
|
||||
const Index patchIdx = patchIndex / m_patchStrides[i];
|
||||
@@ -164,8 +163,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
|
||||
patchOffset -= offsetIdx * m_outputStrides[i];
|
||||
inputIndex += (patchIdx + offsetIdx) * m_inputStrides[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 2; ++i) {
|
||||
const Index patchIdx = patchIndex / m_patchStrides[i];
|
||||
@@ -191,7 +189,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
|
||||
indices[1] - patchIndices[1] * m_outputStrides[output_stride_index]};
|
||||
|
||||
Index inputIndices[2] = {0, 0};
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 2; i > 0; --i) {
|
||||
const Index patchIdx[2] = {patchIndices[0] / m_patchStrides[i], patchIndices[1] / m_patchStrides[i]};
|
||||
@@ -205,8 +203,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> {
|
||||
inputIndices[0] += (patchIdx[0] + offsetIdx[0]) * m_inputStrides[i];
|
||||
inputIndices[1] += (patchIdx[1] + offsetIdx[1]) * m_inputStrides[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 2; ++i) {
|
||||
const Index patchIdx[2] = {patchIndices[0] / m_patchStrides[i], patchIndices[1] / m_patchStrides[i]};
|
||||
|
||||
@@ -607,14 +607,13 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
|
||||
|
||||
// Precompute output strides.
|
||||
if (NumOutputDims > 0) {
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_outputStrides[0] = 1;
|
||||
for (int i = 1; i < NumOutputDims; ++i) {
|
||||
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
|
||||
m_fastOutputStrides[i] = internal::TensorIntDivisor<Index>(m_outputStrides[i]);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_outputStrides[static_cast<size_t>(NumOutputDims - 1)] = 1;
|
||||
for (int i = NumOutputDims - 2; i >= 0; --i) {
|
||||
m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1];
|
||||
@@ -626,13 +625,12 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
|
||||
// Precompute input strides.
|
||||
if (NumInputDims > 0) {
|
||||
array<Index, NumInputDims> input_strides;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
input_strides[0] = 1;
|
||||
for (int i = 1; i < NumInputDims; ++i) {
|
||||
input_strides[i] = input_strides[i - 1] * input_dims[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
input_strides.back() = 1;
|
||||
for (int i = NumInputDims - 2; i >= 0; --i) {
|
||||
input_strides[i] = input_strides[i + 1] * input_dims[i + 1];
|
||||
@@ -695,10 +693,9 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
|
||||
else if ((RunningOnGPU && (m_device.majorDeviceVersion() >= 3)) || (RunningOnSycl)) {
|
||||
bool reducing_inner_dims = true;
|
||||
for (int i = 0; i < NumReducedDims; ++i) {
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
reducing_inner_dims &= m_reduced[i];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
reducing_inner_dims &= m_reduced[NumInputDims - 1 - i];
|
||||
}
|
||||
}
|
||||
@@ -733,10 +730,9 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
|
||||
|
||||
bool preserving_inner_dims = true;
|
||||
for (int i = 0; i < NumReducedDims; ++i) {
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
preserving_inner_dims &= m_reduced[NumInputDims - 1 - i];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
preserving_inner_dims &= m_reduced[i];
|
||||
}
|
||||
}
|
||||
@@ -946,16 +942,15 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
|
||||
// used to compute the reduction at output index "index".
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const {
|
||||
if (ReducingInnerMostDims) {
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
return index * m_preservedStrides[0];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return index * m_preservedStrides[NumPreservedStrides - 1];
|
||||
}
|
||||
}
|
||||
// TBD: optimize the case where we preserve the innermost dimensions.
|
||||
Index startInput = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumOutputDims - 1; i > 0; --i) {
|
||||
// This is index_i in the output tensor.
|
||||
const Index idx = index / m_outputStrides[i];
|
||||
@@ -968,8 +963,7 @@ struct TensorReductionEvaluatorBase<const TensorReductionOp<Op, Dims, ArgType, M
|
||||
} else {
|
||||
startInput += index * m_preservedStrides[0];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumOutputDims - 1; ++i) {
|
||||
// This is index_i in the output tensor.
|
||||
const Index idx = index / m_outputStrides[i];
|
||||
|
||||
@@ -26,7 +26,7 @@ namespace internal {
|
||||
// updated the content of the output address it will try again.
|
||||
template <typename T, typename R>
|
||||
__device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) {
|
||||
EIGEN_IF_CONSTEXPR(sizeof(T) == 4) {
|
||||
EIGEN_IF_CONSTEXPR (sizeof(T) == 4) {
|
||||
unsigned int oldval = *reinterpret_cast<unsigned int*>(output);
|
||||
unsigned int newval = oldval;
|
||||
reducer.reduce(accum, reinterpret_cast<T*>(&newval));
|
||||
@@ -42,8 +42,7 @@ __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer)
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
else EIGEN_IF_CONSTEXPR(sizeof(T) == 8) {
|
||||
} else EIGEN_IF_CONSTEXPR (sizeof(T) == 8) {
|
||||
unsigned long long oldval = *reinterpret_cast<unsigned long long*>(output);
|
||||
unsigned long long newval = oldval;
|
||||
reducer.reduce(accum, reinterpret_cast<T*>(&newval));
|
||||
@@ -59,8 +58,7 @@ __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer)
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
gpu_assert(0 && "Wordsize not supported");
|
||||
}
|
||||
}
|
||||
@@ -367,8 +365,9 @@ template <typename Op>
|
||||
__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void ReductionCleanupKernelHalfFloat(Op reducer, half* output, half* scratch) {
|
||||
eigen_assert(threadIdx.x == 1);
|
||||
typedef packet_traits<Eigen::half>::type packet_type;
|
||||
EIGEN_IF_CONSTEXPR(unpacket_traits<packet_type>::size == 1) { *output = *scratch; }
|
||||
else {
|
||||
EIGEN_IF_CONSTEXPR (unpacket_traits<packet_type>::size == 1) {
|
||||
*output = *scratch;
|
||||
} else {
|
||||
half2* pscratch = reinterpret_cast<half2*>(scratch);
|
||||
half tmp = __float2half(0.f);
|
||||
for (int i = 0; i < unpacket_traits<packet_type>::size; i += 2) {
|
||||
|
||||
@@ -194,13 +194,12 @@ class TensorRefBase : public TensorBase<Derived> {
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(const array<Index, NumIndices>& indices) const {
|
||||
const Dimensions& dims = this->dimensions();
|
||||
Index index = 0;
|
||||
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
|
||||
index += indices[0];
|
||||
for (size_t i = 1; i < NumIndices; ++i) {
|
||||
index = index * dims[i] + indices[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
index += indices[NumIndices - 1];
|
||||
for (int i = NumIndices - 2; i >= 0; --i) {
|
||||
index = index * dims[i] + indices[i];
|
||||
@@ -278,13 +277,12 @@ class TensorRef : public internal::TensorRefBase<TensorRef<PlainObjectType>> {
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(const array<Index, NumIndices>& indices) {
|
||||
const Dimensions& dims = this->dimensions();
|
||||
Index index = 0;
|
||||
EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) {
|
||||
EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) {
|
||||
index += indices[0];
|
||||
for (size_t i = 1; i < NumIndices; ++i) {
|
||||
index = index * dims[i] + indices[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
index += indices[NumIndices - 1];
|
||||
for (int i = NumIndices - 2; i >= 0; --i) {
|
||||
index = index * dims[i] + indices[i];
|
||||
|
||||
@@ -116,14 +116,13 @@ struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device
|
||||
|
||||
// Compute strides
|
||||
m_dimensions = m_impl.dimensions();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_strides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1];
|
||||
if (m_strides[i] > 0) m_fastStrides[i] = IndexDivisor(m_strides[i]);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_strides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1];
|
||||
@@ -151,7 +150,7 @@ struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index reverseIndex(Index index) const {
|
||||
eigen_assert(index < dimensions().TotalSize());
|
||||
Index inputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
Index idx = index / m_fastStrides[i];
|
||||
@@ -166,8 +165,7 @@ struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device
|
||||
} else {
|
||||
inputIndex += index;
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
Index idx = index / m_fastStrides[i];
|
||||
|
||||
@@ -111,14 +111,13 @@ struct TensorEvaluator<const TensorRollOp<RollDimensions, ArgType>, Device> {
|
||||
|
||||
// Compute strides
|
||||
m_dimensions = m_impl.dimensions();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_strides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1];
|
||||
if (m_strides[i] > 0) m_fast_strides[i] = IndexDivisor(m_strides[i]);
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_strides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1];
|
||||
@@ -164,7 +163,7 @@ struct TensorEvaluator<const TensorRollOp<RollDimensions, ArgType>, Device> {
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rollIndex(Index index) const {
|
||||
eigen_assert(index < dimensions().TotalSize());
|
||||
Index rolledIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
Index idx = index / m_fast_strides[i];
|
||||
@@ -172,8 +171,7 @@ struct TensorEvaluator<const TensorRollOp<RollDimensions, ArgType>, Device> {
|
||||
rolledIndex += roll(idx, m_rolls[i], m_dimensions[i]) * m_strides[i];
|
||||
}
|
||||
rolledIndex += roll(index, m_rolls[0], m_dimensions[0]);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
Index idx = index / m_fast_strides[i];
|
||||
@@ -307,15 +305,14 @@ struct TensorEvaluator<const TensorRollOp<RollDimensions, ArgType>, Device> {
|
||||
const Device EIGEN_DEVICE_REF m_device;
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void extract_coordinates(Index index, array<Index, NumDims>& coords) const {
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = index / m_fast_strides[i];
|
||||
index -= idx * m_strides[i];
|
||||
coords[i] = idx;
|
||||
}
|
||||
coords[0] = index;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = index / m_fast_strides[i];
|
||||
index -= idx * m_strides[i];
|
||||
|
||||
@@ -398,12 +398,11 @@ struct TensorEvaluator<const TensorScanOp<Op, ArgType>, Device> {
|
||||
|
||||
// Compute stride of scan axis
|
||||
const Dimensions& dims = m_impl.dimensions();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = 0; i < op.axis(); ++i) {
|
||||
m_stride = m_stride * dims[i];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// dims can only be indexed through unsigned integers,
|
||||
// so use an unsigned type to let the compiler know.
|
||||
// This prevents spurious warnings: "'*((void*)(& evaluator)+64)[18446744073709551615]' may be used uninitialized
|
||||
|
||||
@@ -120,7 +120,7 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_unshuffledInputStrides[0] = 1;
|
||||
m_outputStrides[0] = 1;
|
||||
|
||||
@@ -130,8 +130,7 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
|
||||
m_fastOutputStrides[i] =
|
||||
internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : Index(1));
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_unshuffledInputStrides[NumDims - 1] = 1;
|
||||
m_outputStrides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
@@ -265,15 +264,14 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
|
||||
const DSizes<Index, NumDims>& output_block_strides,
|
||||
const DSizes<internal::TensorIntDivisor<Index>, NumDims>& fast_input_block_strides) const {
|
||||
Index output_index = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = input_index / fast_input_block_strides[i];
|
||||
output_index += idx * output_block_strides[m_inverseShuffle[i]];
|
||||
input_index -= idx * input_block_strides[i];
|
||||
}
|
||||
return output_index + input_index * output_block_strides[m_inverseShuffle[0]];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = input_index / fast_input_block_strides[i];
|
||||
output_index += idx * output_block_strides[m_inverseShuffle[i]];
|
||||
@@ -285,15 +283,14 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
|
||||
Index inputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = index / m_fastOutputStrides[i];
|
||||
inputIndex += idx * m_inputStrides[i];
|
||||
index -= idx * m_outputStrides[i];
|
||||
}
|
||||
return inputIndex + index * m_inputStrides[0];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = index / m_fastOutputStrides[i];
|
||||
inputIndex += idx * m_inputStrides[i];
|
||||
|
||||
@@ -107,7 +107,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
|
||||
}
|
||||
|
||||
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_outputStrides[0] = 1;
|
||||
m_inputStrides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
@@ -116,8 +116,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
|
||||
m_inputStrides[i - 1] *= op.strides()[i - 1];
|
||||
}
|
||||
m_inputStrides[NumDims - 1] *= op.strides()[NumDims - 1];
|
||||
}
|
||||
else { // RowMajor
|
||||
} else { // RowMajor
|
||||
m_outputStrides[NumDims - 1] = 1;
|
||||
m_inputStrides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
@@ -148,7 +147,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
|
||||
|
||||
Index inputIndices[] = {0, 0};
|
||||
Index indices[] = {index, index + PacketSize - 1};
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx0 = indices[0] / m_outputStrides[i];
|
||||
@@ -160,8 +159,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
|
||||
}
|
||||
inputIndices[0] += indices[0] * m_inputStrides[0];
|
||||
inputIndices[1] += indices[1] * m_inputStrides[0];
|
||||
}
|
||||
else { // RowMajor
|
||||
} else { // RowMajor
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx0 = indices[0] / m_outputStrides[i];
|
||||
@@ -208,7 +206,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
|
||||
protected:
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
|
||||
Index inputIndex = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = index / m_outputStrides[i];
|
||||
@@ -216,8 +214,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> {
|
||||
index -= idx * m_outputStrides[i];
|
||||
}
|
||||
inputIndex += index * m_inputStrides[0];
|
||||
}
|
||||
else { // RowMajor
|
||||
} else { // RowMajor
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = index / m_outputStrides[i];
|
||||
@@ -271,7 +268,7 @@ struct TensorEvaluator<TensorStridingOp<Strides, ArgType>, Device>
|
||||
|
||||
Index inputIndices[] = {0, 0};
|
||||
Index indices[] = {index, index + PacketSize - 1};
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx0 = indices[0] / this->m_outputStrides[i];
|
||||
@@ -283,8 +280,7 @@ struct TensorEvaluator<TensorStridingOp<Strides, ArgType>, Device>
|
||||
}
|
||||
inputIndices[0] += indices[0] * this->m_inputStrides[0];
|
||||
inputIndices[1] += indices[1] * this->m_inputStrides[0];
|
||||
}
|
||||
else { // RowMajor
|
||||
} else { // RowMajor
|
||||
EIGEN_UNROLL_LOOP
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx0 = indices[0] / this->m_outputStrides[i];
|
||||
|
||||
@@ -159,13 +159,12 @@ struct TensorEvaluator<const TensorTraceOp<Dims, ArgType>, Device> {
|
||||
|
||||
// Compute the output strides
|
||||
if (NumOutputDims > 0) {
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_outputStrides[0] = 1;
|
||||
for (int i = 1; i < NumOutputDims; ++i) {
|
||||
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_outputStrides.back() = 1;
|
||||
for (int i = NumOutputDims - 2; i >= 0; --i) {
|
||||
m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1];
|
||||
@@ -176,13 +175,12 @@ struct TensorEvaluator<const TensorTraceOp<Dims, ArgType>, Device> {
|
||||
// Compute the input strides
|
||||
if (NumInputDims > 0) {
|
||||
array<Index, NumInputDims> input_strides;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
input_strides[0] = 1;
|
||||
for (int i = 1; i < NumInputDims; ++i) {
|
||||
input_strides[i] = input_strides[i - 1] * input_dims[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
input_strides.back() = 1;
|
||||
for (int i = NumInputDims - 2; i >= 0; --i) {
|
||||
input_strides[i] = input_strides[i + 1] * input_dims[i + 1];
|
||||
@@ -249,15 +247,14 @@ struct TensorEvaluator<const TensorTraceOp<Dims, ArgType>, Device> {
|
||||
// Given the output index, finds the first index in the input tensor used to compute the trace
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const {
|
||||
Index startInput = 0;
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumOutputDims - 1; i > 0; --i) {
|
||||
const Index idx = index / m_outputStrides[i];
|
||||
startInput += idx * m_preservedStrides[i];
|
||||
index -= idx * m_outputStrides[i];
|
||||
}
|
||||
startInput += index * m_preservedStrides[0];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumOutputDims - 1; ++i) {
|
||||
const Index idx = index / m_outputStrides[i];
|
||||
startInput += idx * m_preservedStrides[i];
|
||||
|
||||
@@ -209,13 +209,12 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
|
||||
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
|
||||
|
||||
// Cache a few variables.
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_inputDepth = input_dims[0];
|
||||
m_inputPlanes = input_dims[1];
|
||||
m_inputRows = input_dims[2];
|
||||
m_inputCols = input_dims[3];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_inputDepth = input_dims[NumInputDims - 1];
|
||||
m_inputPlanes = input_dims[NumInputDims - 2];
|
||||
m_inputRows = input_dims[NumInputDims - 3];
|
||||
@@ -288,7 +287,7 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
|
||||
eigen_assert(m_outputPlanes > 0);
|
||||
|
||||
// Dimensions for result of extraction.
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
// ColMajor
|
||||
// 0: depth
|
||||
// 1: patch_planes
|
||||
@@ -304,8 +303,7 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
|
||||
for (int i = 5; i < NumDims; ++i) {
|
||||
m_dimensions[i] = input_dims[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
// RowMajor
|
||||
// NumDims-1: depth
|
||||
// NumDims-2: patch_planes
|
||||
@@ -324,13 +322,12 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
|
||||
}
|
||||
|
||||
// Strides for the output tensor.
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_rowStride = m_dimensions[1];
|
||||
m_colStride = m_dimensions[2] * m_rowStride;
|
||||
m_patchStride = m_colStride * m_dimensions[3] * m_dimensions[0];
|
||||
m_otherStride = m_patchStride * m_dimensions[4];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_rowStride = m_dimensions[NumDims - 2];
|
||||
m_colStride = m_dimensions[NumDims - 3] * m_rowStride;
|
||||
m_patchStride = m_colStride * m_dimensions[NumDims - 4] * m_dimensions[NumDims - 1];
|
||||
@@ -358,10 +355,9 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D
|
||||
m_fastOutputPlanes = internal::TensorIntDivisor<Index>(m_outputPlanes);
|
||||
m_fastOutputPlanesRows = internal::TensorIntDivisor<Index>(m_outputPlanesRows);
|
||||
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[0]);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[NumDims - 1]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,8 +75,8 @@ void sparse_extra(const SparseMatrixType& ref) {
|
||||
// test coeff and coeffRef
|
||||
for (int i = 0; i < (int)zeroCoords.size(); ++i) {
|
||||
VERIFY_IS_MUCH_SMALLER_THAN(m.coeff(zeroCoords[i].x(), zeroCoords[i].y()), eps);
|
||||
EIGEN_IF_CONSTEXPR((std::is_same<SparseMatrixType, SparseMatrix<Scalar, Flags> >::value))
|
||||
VERIFY_RAISES_ASSERT(m.coeffRef(zeroCoords[0].x(), zeroCoords[0].y()) = 5);
|
||||
EIGEN_IF_CONSTEXPR ((std::is_same<SparseMatrixType, SparseMatrix<Scalar, Flags> >::value))
|
||||
VERIFY_RAISES_ASSERT(m.coeffRef(zeroCoords[0].x(), zeroCoords[0].y()) = 5);
|
||||
}
|
||||
VERIFY_IS_APPROX(m, refMat);
|
||||
|
||||
|
||||
@@ -144,15 +144,14 @@ template <int Layout, int NumDims>
|
||||
static Index GetInputIndex(Index output_index, const array<Index, NumDims>& output_to_input_dim_map,
|
||||
const array<Index, NumDims>& input_strides, const array<Index, NumDims>& output_strides) {
|
||||
int input_index = 0;
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = output_index / output_strides[i];
|
||||
input_index += idx * input_strides[output_to_input_dim_map[i]];
|
||||
output_index -= idx * output_strides[i];
|
||||
}
|
||||
return input_index + output_index * input_strides[output_to_input_dim_map[0]];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = output_index / output_strides[i];
|
||||
input_index += idx * input_strides[output_to_input_dim_map[i]];
|
||||
@@ -165,13 +164,12 @@ static Index GetInputIndex(Index output_index, const array<Index, NumDims>& outp
|
||||
template <int Layout, int NumDims>
|
||||
static array<Index, NumDims> ComputeStrides(const array<Index, NumDims>& sizes) {
|
||||
array<Index, NumDims> strides;
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
strides[0] = 1;
|
||||
for (int i = 1; i < NumDims; ++i) {
|
||||
strides[i] = strides[i - 1] * sizes[i - 1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
strides[NumDims - 1] = 1;
|
||||
for (int i = NumDims - 2; i >= 0; --i) {
|
||||
strides[i] = strides[i + 1] * sizes[i + 1];
|
||||
@@ -228,7 +226,7 @@ static void test_uniform_block_shape() {
|
||||
|
||||
// Test shape 'UniformAllDims' with larger 'max_coeff count' which spills
|
||||
// partially into first inner-most dimension.
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 7 * 5 * 5 * 5 * 5;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
|
||||
@@ -238,8 +236,7 @@ static void test_uniform_block_shape() {
|
||||
VERIFY_IS_EQUAL(5, block.dimensions()[i]);
|
||||
}
|
||||
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 5 * 5 * 5 * 5 * 6;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
|
||||
@@ -253,7 +250,7 @@ static void test_uniform_block_shape() {
|
||||
|
||||
// Test shape 'UniformAllDims' with larger 'max_coeff count' which spills
|
||||
// fully into first inner-most dimension.
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 11 * 5 * 5 * 5 * 5;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
|
||||
@@ -263,8 +260,7 @@ static void test_uniform_block_shape() {
|
||||
VERIFY_IS_EQUAL(5, block.dimensions()[i]);
|
||||
}
|
||||
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 5 * 5 * 5 * 5 * 7;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
|
||||
@@ -278,7 +274,7 @@ static void test_uniform_block_shape() {
|
||||
|
||||
// Test shape 'UniformAllDims' with larger 'max_coeff count' which spills
|
||||
// fully into first few inner-most dimensions.
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
DSizes<Index, 5> dims(7, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 7 * 5 * 6 * 7 * 5;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
|
||||
@@ -289,8 +285,7 @@ static void test_uniform_block_shape() {
|
||||
VERIFY_IS_EQUAL(7, block.dimensions()[3]);
|
||||
VERIFY_IS_EQUAL(5, block.dimensions()[4]);
|
||||
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
DSizes<Index, 5> dims(7, 5, 6, 9, 7);
|
||||
const Index max_coeff_count = 5 * 5 * 5 * 6 * 7;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
|
||||
@@ -304,7 +299,7 @@ static void test_uniform_block_shape() {
|
||||
}
|
||||
|
||||
// Test shape 'UniformAllDims' with full allocation to all dims.
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
DSizes<Index, 5> dims(7, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 7 * 5 * 6 * 17 * 7;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
|
||||
@@ -315,8 +310,7 @@ static void test_uniform_block_shape() {
|
||||
VERIFY_IS_EQUAL(17, block.dimensions()[3]);
|
||||
VERIFY_IS_EQUAL(7, block.dimensions()[4]);
|
||||
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
DSizes<Index, 5> dims(7, 5, 6, 9, 7);
|
||||
const Index max_coeff_count = 7 * 5 * 6 * 9 * 7;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()});
|
||||
@@ -336,7 +330,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
typedef internal::TensorBlockMapper<5, Layout> TensorBlockMapper;
|
||||
|
||||
// Test shape 'SkewedInnerDims' with partial allocation to inner-most dim.
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 10 * 1 * 1 * 1 * 1;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
@@ -346,8 +340,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
VERIFY_IS_EQUAL(1, block.dimensions()[i]);
|
||||
}
|
||||
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 1 * 1 * 1 * 1 * 6;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
@@ -360,7 +353,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
}
|
||||
|
||||
// Test shape 'SkewedInnerDims' with full allocation to inner-most dim.
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 11 * 1 * 1 * 1 * 1;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
@@ -370,8 +363,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
VERIFY_IS_EQUAL(1, block.dimensions()[i]);
|
||||
}
|
||||
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 1 * 1 * 1 * 1 * 7;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
@@ -385,7 +377,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
|
||||
// Test shape 'SkewedInnerDims' with full allocation to inner-most dim,
|
||||
// and partial allocation to second inner-dim.
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 11 * 3 * 1 * 1 * 1;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
@@ -396,8 +388,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
VERIFY_IS_EQUAL(1, block.dimensions()[i]);
|
||||
}
|
||||
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 1 * 1 * 1 * 15 * 7;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
@@ -412,7 +403,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
|
||||
// Test shape 'SkewedInnerDims' with full allocation to inner-most dim,
|
||||
// and partial allocation to third inner-dim.
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 11 * 5 * 5 * 1 * 1;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
@@ -424,8 +415,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
VERIFY_IS_EQUAL(1, block.dimensions()[i]);
|
||||
}
|
||||
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 1 * 1 * 5 * 17 * 7;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
@@ -440,7 +430,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
}
|
||||
|
||||
// Test shape 'SkewedInnerDims' with full allocation to all dims.
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 11 * 5 * 6 * 17 * 7;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
@@ -451,8 +441,7 @@ static void test_skewed_inner_dim_block_shape() {
|
||||
VERIFY_IS_EQUAL(17, block.dimensions()[3]);
|
||||
VERIFY_IS_EQUAL(7, block.dimensions()[4]);
|
||||
VERIFY(block.dimensions().TotalSize() <= max_coeff_count);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
DSizes<Index, 5> dims(11, 5, 6, 17, 7);
|
||||
const Index max_coeff_count = 11 * 5 * 6 * 17 * 7;
|
||||
TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()});
|
||||
|
||||
@@ -17,10 +17,9 @@ using Eigen::internal::TensorExecutor;
|
||||
|
||||
template <typename T, int NumDims, int Layout>
|
||||
static void setRandomForBinaryProduct(Tensor<T, NumDims, Layout>& tensor) {
|
||||
EIGEN_IF_CONSTEXPR((std::is_integral<T>::value && !std::is_same<T, bool>::value)) {
|
||||
EIGEN_IF_CONSTEXPR ((std::is_integral<T>::value && !std::is_same<T, bool>::value)) {
|
||||
setRandomDataInRange(tensor, T(-1000), T(1000));
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
tensor.setRandom();
|
||||
}
|
||||
}
|
||||
@@ -86,15 +85,14 @@ static TensorBlockParams<NumDims> SkewedInnerBlock(DSizes<Index, NumDims> dims)
|
||||
|
||||
// Compute offsets for the first block coefficient.
|
||||
Index index = block.offset();
|
||||
EIGEN_IF_CONSTEXPR(static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = index / strides[i];
|
||||
index -= idx * strides[i];
|
||||
offsets[i] = idx;
|
||||
}
|
||||
if (NumDims > 0) offsets[0] = index;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = index / strides[i];
|
||||
index -= idx * strides[i];
|
||||
|
||||
@@ -38,15 +38,14 @@ template <int Layout, int NumDims>
|
||||
static Index GetInputIndex(Index output_index, const array<Index, NumDims>& output_to_input_dim_map,
|
||||
const array<Index, NumDims>& input_strides, const array<Index, NumDims>& output_strides) {
|
||||
int input_index = 0;
|
||||
EIGEN_IF_CONSTEXPR(Layout == ColMajor) {
|
||||
EIGEN_IF_CONSTEXPR (Layout == ColMajor) {
|
||||
for (int i = NumDims - 1; i > 0; --i) {
|
||||
const Index idx = output_index / output_strides[i];
|
||||
input_index += idx * input_strides[output_to_input_dim_map[i]];
|
||||
output_index -= idx * output_strides[i];
|
||||
}
|
||||
return input_index + output_index * input_strides[output_to_input_dim_map[0]];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
for (int i = 0; i < NumDims - 1; ++i) {
|
||||
const Index idx = output_index / output_strides[i];
|
||||
input_index += idx * input_strides[output_to_input_dim_map[i]];
|
||||
|
||||
Reference in New Issue
Block a user