diff --git a/Eigen/src/AccelerateSupport/AccelerateSupport.h b/Eigen/src/AccelerateSupport/AccelerateSupport.h index 4a6748f8c..9739af48f 100644 --- a/Eigen/src/AccelerateSupport/AccelerateSupport.h +++ b/Eigen/src/AccelerateSupport/AccelerateSupport.h @@ -176,30 +176,35 @@ class AccelerateImpl : public SparseSolverBase { mat.coeffRef(i, k) = numext::conj(mat.coeffRef(index_of_biggest_in_corner, i)); mat.coeffRef(index_of_biggest_in_corner, i) = numext::conj(tmp); } - if (NumTraits::IsComplex) - mat.coeffRef(index_of_biggest_in_corner, k) = numext::conj(mat.coeff(index_of_biggest_in_corner, k)); + EIGEN_IF_CONSTEXPR(NumTraits::IsComplex) + mat.coeffRef(index_of_biggest_in_corner, k) = numext::conj(mat.coeff(index_of_biggest_in_corner, k)); } // partition the matrix: diff --git a/Eigen/src/CholmodSupport/CholmodSupport.h b/Eigen/src/CholmodSupport/CholmodSupport.h index 0705d9c86..d5e9ae81c 100644 --- a/Eigen/src/CholmodSupport/CholmodSupport.h +++ b/Eigen/src/CholmodSupport/CholmodSupport.h @@ -83,11 +83,11 @@ cholmod_sparse viewAsCholmod(Ref res.dtype = 0; res.stype = -1; - if (std::is_same::value) { - res.itype = CHOLMOD_INT; - } else if (std::is_same::value) { + EIGEN_IF_CONSTEXPR((std::is_same::value)) { res.itype = CHOLMOD_INT; } + else EIGEN_IF_CONSTEXPR((std::is_same::value)) { res.itype = CHOLMOD_LONG; - } else { + } + else { eigen_assert(false && "Index type not supported yet"); } diff --git a/Eigen/src/Core/ConcatOp.h b/Eigen/src/Core/ConcatOp.h index 71679957c..4ce3cb3b6 100644 --- a/Eigen/src/Core/ConcatOp.h +++ b/Eigen/src/Core/ConcatOp.h @@ -115,9 +115,10 @@ class Concat : public internal::dense_xpr_base> : evaluator_base> : evaluator_base EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { constexpr int packetSize = unpacket_traits::size; - if (Direction == Vertical) { + EIGEN_IF_CONSTEXPR(Direction == Vertical) { const Index boundary = m_lhsRows.value(); if (row >= boundary) return m_rhsImpl.template packet(row - boundary, col); // Column-major: inner=rows, packet extends along rows and may straddle the row boundary. // Row-major: inner=cols, packet extends along cols — never crosses the row boundary. - if (!IsRowMajor && row + packetSize > boundary) return packetBoundary(row, col); + EIGEN_IF_CONSTEXPR(!IsRowMajor) { + if (row + packetSize > boundary) return packetBoundary(row, col); + } return m_lhsImpl.template packet(row, col); - } else { + } + else { const Index boundary = m_lhsCols.value(); if (col >= boundary) return m_rhsImpl.template packet(row, col - boundary); // Row-major: inner=cols, packet extends along cols and may straddle the col boundary. // Column-major: inner=rows, packet extends along rows — never crosses the col boundary. - if (IsRowMajor && col + packetSize > boundary) return packetBoundary(row, col); + EIGEN_IF_CONSTEXPR(IsRowMajor) { + if (col + packetSize > boundary) return packetBoundary(row, col); + } return m_lhsImpl.template packet(row, col); } } @@ -226,19 +233,22 @@ struct evaluator> : evaluator_base EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index row, Index col, Index begin, Index count) const { - if (Direction == Vertical) { + EIGEN_IF_CONSTEXPR(Direction == Vertical) { const Index boundary = m_lhsRows.value(); if (row >= boundary) return m_rhsImpl.template packetSegment(row - boundary, col, begin, count); - if (!IsRowMajor && row + begin + count > boundary) - return packetSegmentBoundary(row, col, begin, count); + EIGEN_IF_CONSTEXPR(!IsRowMajor) { + if (row + begin + count > boundary) return packetSegmentBoundary(row, col, begin, count); + } return m_lhsImpl.template packetSegment(row, col, begin, count); - } else { + } + else { const Index boundary = m_lhsCols.value(); if (col >= boundary) return m_rhsImpl.template packetSegment(row, col - boundary, begin, count); - if (IsRowMajor && col + begin + count > boundary) - return packetSegmentBoundary(row, col, begin, count); + EIGEN_IF_CONSTEXPR(IsRowMajor) { + if (col + begin + count > boundary) return packetSegmentBoundary(row, col, begin, count); + } return m_lhsImpl.template packetSegment(row, col, begin, count); } } diff --git a/Eigen/src/Core/CwiseNullaryOp.h b/Eigen/src/Core/CwiseNullaryOp.h index d0bc9bb6f..0241276ec 100644 --- a/Eigen/src/Core/CwiseNullaryOp.h +++ b/Eigen/src/Core/CwiseNullaryOp.h @@ -142,10 +142,10 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE #endif DenseBase::NullaryExpr(Index size, const CustomNullaryOp& func) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - if (RowsAtCompileTime == 1) - return CwiseNullaryOp(1, size, func); - else + EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { return CwiseNullaryOp(1, size, func); } + else { return CwiseNullaryOp(size, 1, func); + } } /** \returns an expression of a matrix defined by a custom functor \a func diff --git a/Eigen/src/Core/CwiseTernaryOp.h b/Eigen/src/Core/CwiseTernaryOp.h index dcb62d245..0a33cdb1c 100644 --- a/Eigen/src/Core/CwiseTernaryOp.h +++ b/Eigen/src/Core/CwiseTernaryOp.h @@ -122,26 +122,32 @@ class CwiseTernaryOp : public CwiseTernaryOpImpl>::RowsAtCompileTime == Dynamic && - internal::traits>::RowsAtCompileTime == Dynamic) + EIGEN_IF_CONSTEXPR(internal::traits>::RowsAtCompileTime == Dynamic && + internal::traits>::RowsAtCompileTime == Dynamic) { return m_arg3.rows(); - else if (internal::traits>::RowsAtCompileTime == Dynamic && - internal::traits>::RowsAtCompileTime == Dynamic) + } + else EIGEN_IF_CONSTEXPR(internal::traits>::RowsAtCompileTime == Dynamic && + internal::traits>::RowsAtCompileTime == Dynamic) { return m_arg2.rows(); - else + } + else { return m_arg1.rows(); + } } EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Index cols() const { // return the fixed size type if available to enable compile time // optimizations - if (internal::traits>::ColsAtCompileTime == Dynamic && - internal::traits>::ColsAtCompileTime == Dynamic) + EIGEN_IF_CONSTEXPR(internal::traits>::ColsAtCompileTime == Dynamic && + internal::traits>::ColsAtCompileTime == Dynamic) { return m_arg3.cols(); - else if (internal::traits>::ColsAtCompileTime == Dynamic && - internal::traits>::ColsAtCompileTime == Dynamic) + } + else EIGEN_IF_CONSTEXPR(internal::traits>::ColsAtCompileTime == Dynamic && + internal::traits>::ColsAtCompileTime == Dynamic) { return m_arg2.cols(); - else + } + else { return m_arg1.cols(); + } } /** \returns the first argument nested expression */ diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index 3108e8781..3877ba379 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h @@ -1448,10 +1448,10 @@ inline void pstore1(typename unpacket_traits::type* to, const typename u * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_traits::type* from) { - if (Alignment >= unpacket_traits::alignment) - return pload(from); - else + EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits::alignment) { return pload(from); } + else { return ploadu(from); + } } /** \internal \returns n elements of a packet version of \a *from. @@ -1459,20 +1459,20 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_trai template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(const typename unpacket_traits::type* from, const Index n, const Index offset = 0) { - if (Alignment >= unpacket_traits::alignment) - return pload_partial(from, n, offset); - else + EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits::alignment) { return pload_partial(from, n, offset); } + else { return ploadu_partial(from, n, offset); + } } /** \internal copy the packet \a from to \a *to. * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& from) { - if (Alignment >= unpacket_traits::alignment) - pstore(to, from); - else + EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits::alignment) { pstore(to, from); } + else { pstoreu(to, from); + } } /** \internal copy n elements of the packet \a from to \a *to. @@ -1480,10 +1480,10 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& fro template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret_partial(Scalar* to, const Packet& from, const Index n, const Index offset = 0) { - if (Alignment >= unpacket_traits::alignment) - pstore_partial(to, from, n, offset); - else + EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits::alignment) { pstore_partial(to, from, n, offset); } + else { pstoreu_partial(to, from, n, offset); + } } /** \internal \returns a packet version of \a *from. @@ -1677,9 +1677,8 @@ template EIGEN_DEVICE_FUNC inline Packet ploadtSegment(const typename unpacket_traits::type* from, Index begin, Index count) { constexpr int RequiredAlignment = unpacket_traits::alignment; - if (Alignment >= RequiredAlignment) { - return ploadSegment(from, begin, count); - } else { + EIGEN_IF_CONSTEXPR(Alignment >= RequiredAlignment) { return ploadSegment(from, begin, count); } + else { return ploaduSegment(from, begin, count); } } @@ -1689,9 +1688,8 @@ Elements outside of the range [begin, begin + count) are not defined.*/ template EIGEN_DEVICE_FUNC inline void pstoretSegment(Scalar* to, const Packet& from, Index begin, Index count) { constexpr int RequiredAlignment = unpacket_traits::alignment; - if (Alignment >= RequiredAlignment) { - pstoreSegment(to, from, begin, count); - } else { + EIGEN_IF_CONSTEXPR(Alignment >= RequiredAlignment) { pstoreSegment(to, from, begin, count); } + else { pstoreuSegment(to, from, begin, count); } } diff --git a/Eigen/src/Core/IO.h b/Eigen/src/Core/IO.h index 8484f2b46..faeeb9f5b 100644 --- a/Eigen/src/Core/IO.h +++ b/Eigen/src/Core/IO.h @@ -150,9 +150,8 @@ std::ostream& print_matrix(std::ostream& s, const Derived& _m, const IOFormat& f if (fmt.precision == StreamPrecision) { explicit_precision = 0; } else if (fmt.precision == FullPrecision) { - if (NumTraits::IsInteger) { - explicit_precision = 0; - } else { + EIGEN_IF_CONSTEXPR(NumTraits::IsInteger) { explicit_precision = 0; } + else { explicit_precision = significant_decimals_impl::run(); } } else { diff --git a/Eigen/src/Core/IndexedView.h b/Eigen/src/Core/IndexedView.h index 0f6d83cf0..1ecaf9ad9 100644 --- a/Eigen/src/Core/IndexedView.h +++ b/Eigen/src/Core/IndexedView.h @@ -199,13 +199,13 @@ class IndexedViewImpl IndexedViewImpl(XprType& xpr, const T0& rowIndices, const T1& colIndices) : Base(xpr, rowIndices, colIndices) {} Index rowIncrement() const { - if (traits::RowIncr != DynamicIndex && traits::RowIncr != Undefined) { + EIGEN_IF_CONSTEXPR(traits::RowIncr != DynamicIndex && traits::RowIncr != Undefined) { return traits::RowIncr; } return IndexedViewHelper::incr(this->rowIndices()); } Index colIncrement() const { - if (traits::ColIncr != DynamicIndex && traits::ColIncr != Undefined) { + EIGEN_IF_CONSTEXPR(traits::ColIncr != DynamicIndex && traits::ColIncr != Undefined) { return traits::ColIncr; } return IndexedViewHelper::incr(this->colIndices()); @@ -228,14 +228,14 @@ class IndexedViewImpl } EIGEN_DEVICE_FUNC constexpr Index innerStride() const noexcept { - if (traits::InnerStrideAtCompileTime != Dynamic) { + EIGEN_IF_CONSTEXPR(traits::InnerStrideAtCompileTime != Dynamic) { return traits::InnerStrideAtCompileTime; } return innerIncrement() * this->nestedExpression().innerStride(); } EIGEN_DEVICE_FUNC constexpr Index outerStride() const noexcept { - if (traits::OuterStrideAtCompileTime != Dynamic) { + EIGEN_IF_CONSTEXPR(traits::OuterStrideAtCompileTime != Dynamic) { return traits::OuterStrideAtCompileTime; } return outerIncrement() * this->nestedExpression().outerStride(); diff --git a/Eigen/src/Core/MapBase.h b/Eigen/src/Core/MapBase.h index a9cb690ba..9f08abae4 100644 --- a/Eigen/src/Core/MapBase.h +++ b/Eigen/src/Core/MapBase.h @@ -177,7 +177,7 @@ class MapBase : public internal::dense_xpr_base 0 // innerStride() is not set yet when this function is called, so we optimistically assume the lowest plausible // value: - const Index minInnerStride = InnerStrideAtCompileTime == Dynamic ? 1 : Index(InnerStrideAtCompileTime); + constexpr Index minInnerStride = InnerStrideAtCompileTime == Dynamic ? 1 : Index(InnerStrideAtCompileTime); EIGEN_ONLY_USED_FOR_DEBUG(minInnerStride); eigen_assert((((std::uintptr_t(m_data) % internal::traits::Alignment) == 0) || (cols() * rows() * minInnerStride * sizeof(Scalar)) < internal::traits::Alignment) && diff --git a/Eigen/src/Core/PlainObjectBase.h b/Eigen/src/Core/PlainObjectBase.h index da02f8490..1963fd023 100644 --- a/Eigen/src/Core/PlainObjectBase.h +++ b/Eigen/src/Core/PlainObjectBase.h @@ -171,10 +171,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type { * * See DenseCoeffsBase::coeff(Index) const for details. */ EIGEN_DEVICE_FUNC constexpr const Scalar& coeff(Index rowId, Index colId) const { - if (Flags & RowMajorBit) - return m_storage.data()[colId + rowId * m_storage.cols()]; - else // column-major + EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; } + else { // column-major return m_storage.data()[rowId + colId * m_storage.rows()]; + } } /** This is an overloaded version of DenseCoeffsBase::coeff(Index) const @@ -188,10 +188,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type { * * See DenseCoeffsBase::coeffRef(Index,Index) const for details. */ EIGEN_DEVICE_FUNC constexpr Scalar& coeffRef(Index rowId, Index colId) { - if (Flags & RowMajorBit) - return m_storage.data()[colId + rowId * m_storage.cols()]; - else // column-major + EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; } + else { // column-major return m_storage.data()[rowId + colId * m_storage.rows()]; + } } /** This is an overloaded version of DenseCoeffsBase::coeffRef(Index) const @@ -203,10 +203,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type { /** This is the const version of coeffRef(Index,Index) which is thus synonym of coeff(Index,Index). * It is provided for convenience. */ EIGEN_DEVICE_FUNC constexpr const Scalar& coeffRef(Index rowId, Index colId) const { - if (Flags & RowMajorBit) - return m_storage.data()[colId + rowId * m_storage.cols()]; - else // column-major + EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; } + else { // column-major return m_storage.data()[rowId + colId * m_storage.rows()]; + } } /** This is the const version of coeffRef(Index) which is thus synonym of coeff(Index). @@ -303,10 +303,10 @@ class PlainObjectBase : public internal::dense_xpr_base::type { #ifdef EIGEN_INITIALIZE_COEFFS bool size_changed = size != this->size(); #endif - if (RowsAtCompileTime == 1) - m_storage.resize(size, 1, size); - else + EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { m_storage.resize(size, 1, size); } + else { m_storage.resize(size, size, 1); + } #ifdef EIGEN_INITIALIZE_COEFFS if (size_changed) EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED #endif @@ -347,14 +347,17 @@ class PlainObjectBase : public internal::dense_xpr_base::type { other.rows(), other.cols()); #endif const Index othersize = other.rows() * other.cols(); - if (RowsAtCompileTime == 1) { + EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { eigen_assert(other.rows() == 1 || other.cols() == 1); resize(1, othersize); - } else if (ColsAtCompileTime == 1) { + } + else EIGEN_IF_CONSTEXPR(ColsAtCompileTime == 1) { eigen_assert(other.rows() == 1 || other.cols() == 1); resize(othersize, 1); - } else + } + else { resize(other.rows(), other.cols()); + } } /** Resizes the matrix to \a rows x \a cols while leaving old values untouched. diff --git a/Eigen/src/Core/ProductEvaluators.h b/Eigen/src/Core/ProductEvaluators.h index 273082420..7af29111e 100644 --- a/Eigen/src/Core/ProductEvaluators.h +++ b/Eigen/src/Core/ProductEvaluators.h @@ -899,15 +899,18 @@ struct triangular_diagonal_product_impl { const Index rows = matrix.rows(); const Index cols = matrix.cols(); for (Index col = 0; col < cols; ++col) { - if ((Mode & Upper) == Upper) { + EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) { const Index end = (std::min)(rows, ((Mode & (UnitDiag | ZeroDiag)) ? col : col + 1)); addStoredSegment(dst, matrix, diagonal, 0, end, col, alpha); - } else { + } + else { const Index begin = ((Mode & (UnitDiag | ZeroDiag)) ? col + 1 : col); addStoredSegment(dst, matrix, diagonal, begin, rows - begin, col, alpha); } - if ((Mode & UnitDiag) == UnitDiag && col < rows) addUnitCoeff(dst, diagonal, col, alpha); + EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) { + if (col < rows) addUnitCoeff(dst, diagonal, col, alpha); + } } } @@ -925,10 +928,12 @@ struct triangular_diagonal_product_impl { template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addUnitCoeff(Dest& dst, const DiagonalType& diagonal, Index index, const Alpha& alpha) { - if (ProductOrder == OnTheLeft) + EIGEN_IF_CONSTEXPR(ProductOrder == OnTheLeft) { dst.coeffRef(index, index) += alpha * (diagonal.coeff(index) * MatrixScalar(1)); - else + } + else { dst.coeffRef(index, index) += alpha * (MatrixScalar(1) * diagonal.coeff(index)); + } } }; @@ -1036,9 +1041,10 @@ struct selfadjoint_diagonal_product_impl { // Stored half: one column-strided segment per output column. for (Index col = 0; col < size; ++col) { - if ((Mode & Upper) == Upper) { + EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) { storedSegment(dst, matrix, diagonal, 0, col + 1, col, alpha); - } else { + } + else { storedSegment(dst, matrix, diagonal, col, size - col, col, alpha); } } @@ -1047,7 +1053,7 @@ struct selfadjoint_diagonal_product_impl { for (Index ib = 0; ib < size; ib += BlockSize) { const Index ib_end = numext::mini(size, ib + BlockSize); const Index br = ib_end - ib; - if ((Mode & Upper) == Upper) { + EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) { // Off-diagonal: write strict-lower of dst from strict-upper of source. for (Index jb = 0; jb < ib; jb += BlockSize) { const Index bc = numext::mini(jb + BlockSize, ib) - jb; @@ -1056,7 +1062,8 @@ struct selfadjoint_diagonal_product_impl { // Diagonal tile: in-tile strict-lower mirror. for (Index col = ib; col < ib_end; ++col) conjugateSegment(dst, matrix, diagonal, col + 1, ib_end - col - 1, col, alpha); - } else { + } + else { // Off-diagonal: write strict-upper of dst from strict-lower of source. for (Index jb = ib_end; jb < size; jb += BlockSize) { const Index bc = numext::mini(size, jb + BlockSize) - jb; @@ -1076,10 +1083,12 @@ struct selfadjoint_diagonal_product_impl { if (size <= 0) return; auto dstSegment = dst.col(col).segment(begin, size); auto srcSegment = matrix.col(col).segment(begin, size); - if (Accumulate) + EIGEN_IF_CONSTEXPR(Accumulate) { diagonal_product_segment_impl::run(dstSegment, srcSegment, diagonal, begin, col, alpha); - else + } + else { diagonal_product_segment_impl::runOverwrite(dstSegment, srcSegment, diagonal, begin, col); + } } template @@ -1089,10 +1098,12 @@ struct selfadjoint_diagonal_product_impl { if (size <= 0) return; auto dstSegment = dst.col(col).segment(begin, size); auto srcSegment = matrix.row(col).segment(begin, size).conjugate().transpose(); - if (Accumulate) + EIGEN_IF_CONSTEXPR(Accumulate) { diagonal_product_segment_impl::run(dstSegment, srcSegment, diagonal, begin, col, alpha); - else + } + else { diagonal_product_segment_impl::runOverwrite(dstSegment, srcSegment, diagonal, begin, col); + } } // dst.block(ib, jb, br, bc) [+= alpha *] matrix.block(jb, ib, bc, br).adjoint() * , @@ -1104,18 +1115,19 @@ struct selfadjoint_diagonal_product_impl { Index br, Index bc, const Alpha& alpha) { auto dstBlock = dst.block(ib, jb, br, bc); auto srcAdjoint = matrix.block(jb, ib, bc, br).adjoint(); - if (ProductOrder == OnTheRight) { + EIGEN_IF_CONSTEXPR(ProductOrder == OnTheRight) { auto scaled = srcAdjoint * diagonal.segment(jb, bc).asDiagonal(); - if (Accumulate) - dstBlock.noalias() += alpha * scaled; - else + EIGEN_IF_CONSTEXPR(Accumulate) { dstBlock.noalias() += alpha * scaled; } + else { dstBlock.noalias() = scaled; - } else { + } + } + else { auto scaled = diagonal.segment(ib, br).asDiagonal() * srcAdjoint; - if (Accumulate) - dstBlock.noalias() += alpha * scaled; - else + EIGEN_IF_CONSTEXPR(Accumulate) { dstBlock.noalias() += alpha * scaled; } + else { dstBlock.noalias() = scaled; + } } } }; @@ -1168,12 +1180,13 @@ struct generic_product_impl static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) { - if (LhsBlasTraits::HasScalarFactor) { + EIGEN_IF_CONSTEXPR(LhsBlasTraits::HasScalarFactor) { // Folded scalar factor present: zero dst then accumulate at the extracted alpha. Scalar factor = LhsBlasTraits::extractScalarFactor(lhs.nestedExpression()); dst.setZero(); Kernel::run(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal(), factor); - } else { + } + else { // No scalar factor: kernel writes every entry exactly once, skip setZero. Kernel::runOverwrite(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal()); } @@ -1206,11 +1219,12 @@ struct generic_product_impl static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) { - if (RhsBlasTraits::HasScalarFactor) { + EIGEN_IF_CONSTEXPR(RhsBlasTraits::HasScalarFactor) { Scalar factor = RhsBlasTraits::extractScalarFactor(rhs.nestedExpression()); dst.setZero(); Kernel::run(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal(), factor); - } else { + } + else { Kernel::runOverwrite(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal()); } } @@ -1457,11 +1471,11 @@ struct triangular_diagonal_product_lazy_evaluator_base : evaluator_base const bool inActive = ((Mode & Upper) == Upper) ? (row <= col) : (row >= col); if (!inActive) return Scalar(0); if (row == col) { - if ((Mode & UnitDiag) == UnitDiag) { + EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) { return ProductOrder == OnTheLeft ? Scalar(m_diagImpl.coeff(row) * MatrixScalar(1)) : Scalar(MatrixScalar(1) * m_diagImpl.coeff(col)); } - if ((Mode & ZeroDiag) == ZeroDiag) return Scalar(0); + EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag) return Scalar(0); } return ProductOrder == OnTheLeft ? Scalar(m_diagImpl.coeff(row) * m_matImpl.coeff(row, col)) : Scalar(m_matImpl.coeff(row, col) * m_diagImpl.coeff(col)); @@ -1687,10 +1701,10 @@ struct transposition_matrix_product { for (Index k = (Transposed ? size - 1 : 0); Transposed ? k >= 0 : k < size; Transposed ? --k : ++k) if (Index(j = tr.coeff(k)) != k) { - if (Side == OnTheLeft) - dst.row(k).swap(dst.row(j)); - else if (Side == OnTheRight) + EIGEN_IF_CONSTEXPR(Side == OnTheLeft) { dst.row(k).swap(dst.row(j)); } + else EIGEN_IF_CONSTEXPR(Side == OnTheRight) { dst.col(k).swap(dst.col(j)); + } } } }; diff --git a/Eigen/src/Core/Ref.h b/Eigen/src/Core/Ref.h index d2a4647e0..b39863049 100644 --- a/Eigen/src/Core/Ref.h +++ b/Eigen/src/Core/Ref.h @@ -126,11 +126,12 @@ class RefBase : public MapBase { // Determine runtime rows and columns. Index rows = expr.rows(); Index cols = expr.cols(); - if (PlainObjectType::RowsAtCompileTime == 1) { + EIGEN_IF_CONSTEXPR(PlainObjectType::RowsAtCompileTime == 1) { eigen_assert(expr.rows() == 1 || expr.cols() == 1); rows = 1; cols = expr.size(); - } else if (PlainObjectType::ColsAtCompileTime == 1) { + } + else EIGEN_IF_CONSTEXPR(PlainObjectType::ColsAtCompileTime == 1) { eigen_assert(expr.rows() == 1 || expr.cols() == 1); rows = expr.size(); cols = 1; diff --git a/Eigen/src/Core/Reshaped.h b/Eigen/src/Core/Reshaped.h index 3762103d8..c9e75186d 100644 --- a/Eigen/src/Core/Reshaped.h +++ b/Eigen/src/Core/Reshaped.h @@ -296,10 +296,11 @@ struct reshaped_evaluator RowCol; EIGEN_DEVICE_FUNC constexpr inline RowCol index_remap(Index rowId, Index colId) const { - if (Order == ColMajor) { + EIGEN_IF_CONSTEXPR(Order == ColMajor) { const Index nth_elem_idx = colId * m_xpr.rows() + rowId; return RowCol(nth_elem_idx % m_xpr.nestedExpression().rows(), nth_elem_idx / m_xpr.nestedExpression().rows()); - } else { + } + else { const Index nth_elem_idx = colId + rowId * m_xpr.cols(); return RowCol(nth_elem_idx / m_xpr.nestedExpression().cols(), nth_elem_idx % m_xpr.nestedExpression().cols()); } diff --git a/Eigen/src/Core/SelfAdjointView.h b/Eigen/src/Core/SelfAdjointView.h index 3fc75a85f..cb076b73a 100644 --- a/Eigen/src/Core/SelfAdjointView.h +++ b/Eigen/src/Core/SelfAdjointView.h @@ -219,12 +219,14 @@ class SelfAdjointView : public TriangularBase const Index n = m_matrix.rows(); for (Index col = 0; col < n; ++col) { RealScalar_ abs_col_sum; - if (UpLo == Lower) + EIGEN_IF_CONSTEXPR(UpLo == Lower) { abs_col_sum = m_matrix.col(col).tail(n - col).template lpNorm<1>() + m_matrix.row(col).head(col).template lpNorm<1>(); - else + } + else { abs_col_sum = m_matrix.col(col).head(col).template lpNorm<1>() + m_matrix.row(col).tail(n - col).template lpNorm<1>(); + } if (abs_col_sum > norm) norm = abs_col_sum; } return norm; diff --git a/Eigen/src/Core/TriangularMatrix.h b/Eigen/src/Core/TriangularMatrix.h index 065323d9d..a4fbf98a5 100644 --- a/Eigen/src/Core/TriangularMatrix.h +++ b/Eigen/src/Core/TriangularMatrix.h @@ -391,12 +391,13 @@ class TriangularView /** \returns the determinant of the triangular matrix * \sa MatrixBase::determinant() */ EIGEN_DEVICE_FUNC Scalar determinant() const { - if (Mode & UnitDiag) - return 1; - else if (Mode & ZeroDiag) + EIGEN_IF_CONSTEXPR(Mode & UnitDiag) { return 1; } + else EIGEN_IF_CONSTEXPR(Mode & ZeroDiag) { return 0; - else + } + else { return m_matrix.diagonal().prod(); + } } protected: @@ -776,17 +777,18 @@ class triangular_dense_assignment_kernel #endif EIGEN_DEVICE_FUNC void assignDiagonalCoeff(Index id) { - if (Mode == UnitDiag && SetOpposite) - m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(1)); - else if (Mode == ZeroDiag && SetOpposite) + EIGEN_IF_CONSTEXPR(Mode == UnitDiag && SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(1)); } + else EIGEN_IF_CONSTEXPR(Mode == ZeroDiag && SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(0)); - else if (Mode == 0) + } + else EIGEN_IF_CONSTEXPR(Mode == 0) { Base::assignCoeff(id, id); + } } EIGEN_DEVICE_FUNC void assignOppositeCoeff(Index row, Index col) { eigen_internal_assert(row != col); - if (SetOpposite) m_functor.assignCoeff(m_dst.coeffRef(row, col), Scalar(0)); + EIGEN_IF_CONSTEXPR(SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(row, col), Scalar(0)); } } }; @@ -892,8 +894,8 @@ struct triangular_assignment_loop { kernel.assignDiagonalCoeff(row); else if (((Mode & Lower) && row > col) || ((Mode & Upper) && row < col)) kernel.assignCoeff(row, col); - else if (SetOpposite) - kernel.assignOppositeCoeff(row, col); + else + EIGEN_IF_CONSTEXPR(SetOpposite) { kernel.assignOppositeCoeff(row, col); } } }; @@ -940,19 +942,22 @@ struct triangular_assignment_loop { const Index maxi = numext::mini(outer, innerSize); Index i = 0; - if (ActiveBeforeDiag) { + EIGEN_IF_CONSTEXPR(ActiveBeforeDiag) { for (; i < maxi; ++i) kernel.assignCoeff(row(outer, i), col(outer, i)); - } else if (SetOpposite) { + } + else EIGEN_IF_CONSTEXPR(SetOpposite) { for (; i < maxi; ++i) kernel.assignOppositeCoeff(row(outer, i), col(outer, i)); - } else { + } + else { i = maxi; } if (i < innerSize) kernel.assignDiagonalCoeff(i++); - if (!ActiveBeforeDiag) { + EIGEN_IF_CONSTEXPR(!ActiveBeforeDiag) { for (; i < innerSize; ++i) kernel.assignCoeff(row(outer, i), col(outer, i)); - } else if (SetOpposite) { + } + else EIGEN_IF_CONSTEXPR(SetOpposite) { for (; i < innerSize; ++i) kernel.assignOppositeCoeff(row(outer, i), col(outer, i)); } } diff --git a/Eigen/src/Core/arch/AVX512/GemmKernel.h b/Eigen/src/Core/arch/AVX512/GemmKernel.h index 5e8c045ab..6855ea8fe 100644 --- a/Eigen/src/Core/arch/AVX512/GemmKernel.h +++ b/Eigen/src/Core/arch/AVX512/GemmKernel.h @@ -1001,10 +1001,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= 4 ? (cols / 4) * 4 : 0; Index count = 0; const Index peeled_k = (depth / PacketSize) * PacketSize; - if (nr >= 8) { + EIGEN_IF_CONSTEXPR(nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { // skip what we have before - if (PanelMode) count += 8 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset; const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); @@ -1079,14 +1079,14 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= 4) { + EIGEN_IF_CONSTEXPR(nr >= 4) { for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { // skip what we have before - if (PanelMode) count += 4 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset; const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); @@ -1133,19 +1133,19 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= depth && offset <= stride)); - const bool HasHalf = (int)HalfPacketSize < (int)PacketSize; - const bool HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize; + constexpr bool HasHalf = (int)HalfPacketSize < (int)PacketSize; + constexpr bool HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize; conj_if::IsComplex && Conjugate> cj; Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0; Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0; Index count = 0; - if (nr >= 8) { + EIGEN_IF_CONSTEXPR(nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { // skip what we have before - if (PanelMode) count += 8 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset; for (Index k = 0; k < depth; k++) { - if (PacketSize == 8) { + EIGEN_IF_CONSTEXPR(PacketSize == 8) { // Packet A = ploadu(&rhs.data()[k*rhs.stride() + j2]); Packet A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); - } else if (HasHalf && HalfPacketSize == 8) { + } + else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 8) { HalfPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); - } else if (HasQuarter && QuarterPacketSize == 8) { + } + else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 8) { QuarterPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); - } else if (PacketSize == 4) { + } + else EIGEN_IF_CONSTEXPR(PacketSize == 4) { // Packet A = ploadu(&rhs.data()[k*rhs.stride() + j2]); // Packet B = ploadu(&rhs.data()[k*rhs.stride() + j2 + PacketSize]); Packet A = rhs.template loadPacket(k, j2); Packet B = rhs.template loadPacket(k, j2 + PacketSize); pstoreu(blockB + count, cj.pconj(A)); pstoreu(blockB + count + PacketSize, cj.pconj(B)); - } else { + } + else { // const Scalar* b0 = &rhs.data()[k*rhs.stride() + j2]; const LinearMapper dm0 = rhs.getLinearMapper(k, j2); blockB[count + 0] = cj(dm0(0)); @@ -1211,28 +1215,31 @@ struct gemm_pack_rhs= 4) { + EIGEN_IF_CONSTEXPR(nr >= 4) { for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { // skip what we have before - if (PanelMode) count += 4 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset; for (Index k = 0; k < depth; k++) { - if (PacketSize == 4) { + EIGEN_IF_CONSTEXPR(PacketSize == 4) { Packet A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += PacketSize; - } else if (HasHalf && HalfPacketSize == 4) { + } + else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 4) { HalfPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += HalfPacketSize; - } else if (HasQuarter && QuarterPacketSize == 4) { + } + else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 4) { QuarterPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += QuarterPacketSize; - } else { + } + else { const LinearMapper dm0 = rhs.getLinearMapper(k, j2); blockB[count + 0] = cj(dm0(0)); blockB[count + 1] = cj(dm0(1)); @@ -1242,17 +1249,17 @@ struct gemm_pack_rhs& block) { const Index size = 16 / sizeof(Scalar); pstore(to + (0 * size), block.packet[0]); pstore(to + (1 * size), block.packet[1]); - if (N > 2) { - pstore(to + (2 * size), block.packet[2]); - } - if (N > 3) { - pstore(to + (3 * size), block.packet[3]); - } + EIGEN_IF_CONSTEXPR(N > 2) { pstore(to + (2 * size), block.packet[2]); } + EIGEN_IF_CONSTEXPR(N > 3) { pstore(to + (3 * size), block.packet[3]); } } // General template for lhs & rhs complex packing. @@ -424,21 +420,21 @@ struct dhs_cpack { PacketBlock cblock; for (; i + vectorSize <= depth; i += vectorSize) { - if (UseLhs) { - bload(cblock, lhs2, 0, i); - } else { + EIGEN_IF_CONSTEXPR(UseLhs) { bload(cblock, lhs2, 0, i); } + else { bload(cblock, lhs2, i, 0); } - if (((StorageOrder == RowMajor) && UseLhs) || (((StorageOrder == ColMajor) && !UseLhs))) { + EIGEN_IF_CONSTEXPR(((StorageOrder == RowMajor) && UseLhs) || (((StorageOrder == ColMajor) && !UseLhs))) { dhs_cblock(cblock, blockr, p16uc_GETREAL32b); dhs_cblock(cblock, blocki, p16uc_GETIMAG32b); - } else { + } + else { dhs_cblock(cblock, blockr, p16uc_GETREAL32); dhs_cblock(cblock, blocki, p16uc_GETIMAG32); } - if (Conjugate) { + EIGEN_IF_CONSTEXPR(Conjugate) { blocki.packet[0] = -blocki.packet[0]; blocki.packet[1] = -blocki.packet[1]; blocki.packet[2] = -blocki.packet[2]; @@ -473,19 +469,22 @@ struct dhs_cpack { PacketBlock blockr, blocki; PacketBlock cblock; - if (((StorageOrder == ColMajor) && UseLhs) || (((StorageOrder == RowMajor) && !UseLhs))) { - if (UseLhs) { + EIGEN_IF_CONSTEXPR(((StorageOrder == ColMajor) && UseLhs) || (((StorageOrder == RowMajor) && !UseLhs))) { + EIGEN_IF_CONSTEXPR(UseLhs) { cblock.packet[0] = lhs2.template loadPacket(0, i); cblock.packet[1] = lhs2.template loadPacket(2, i); - } else { + } + else { cblock.packet[0] = lhs2.template loadPacket(i, 0); cblock.packet[1] = lhs2.template loadPacket(i, 2); } - } else { - if (UseLhs) { + } + else { + EIGEN_IF_CONSTEXPR(UseLhs) { cblock.packet[0] = pload2(lhs2(0, i), lhs2(1, i)); cblock.packet[1] = pload2(lhs2(2, i), lhs2(3, i)); - } else { + } + else { cblock.packet[0] = pload2(lhs2(i, 0), lhs2(i, 1)); cblock.packet[1] = pload2(lhs2(i, 2), lhs2(i, 3)); } @@ -494,9 +493,7 @@ struct dhs_cpack { blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL32); blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG32); - if (Conjugate) { - blocki.packet[0] = -blocki.packet[0]; - } + EIGEN_IF_CONSTEXPR(Conjugate) { blocki.packet[0] = -blocki.packet[0]; } pstore(blockAt + rir, blockr.packet[0]); pstore(blockAt + rii, blocki.packet[0]); @@ -508,8 +505,8 @@ struct dhs_cpack { rir += ((PanelMode) ? (vectorSize * (2 * stride - depth)) : vectorDelta); } - if (!UseLhs) { - if (PanelMode) rir -= (offset * (vectorSize - 1)); + EIGEN_IF_CONSTEXPR(!UseLhs) { + EIGEN_IF_CONSTEXPR(PanelMode) rir -= (offset * (vectorSize - 1)); for (; j < rows; j++) { const DataMapper lhs2 = lhs.getSubMapper(0, j); @@ -518,10 +515,9 @@ struct dhs_cpack { for (Index i = 0; i < depth; i++) { blockAt[rir] = lhs2(i, 0).real(); - if (Conjugate) - blockAt[rii] = -lhs2(i, 0).imag(); - else - blockAt[rii] = lhs2(i, 0).imag(); + EIGEN_IF_CONSTEXPR(Conjugate) + blockAt[rii] = -lhs2(i, 0).imag(); + else blockAt[rii] = lhs2(i, 0).imag(); rir += 1; rii += 1; @@ -529,9 +525,10 @@ struct dhs_cpack { rir += ((PanelMode) ? (2 * stride - depth) : depth); } - } else { + } + else { if (j < rows) { - if (PanelMode) rir += (offset * (rows - j - vectorSize)); + EIGEN_IF_CONSTEXPR(PanelMode) rir += (offset * (rows - j - vectorSize)); rii = rir + (((PanelMode) ? stride : depth) * (rows - j)); for (Index i = 0; i < depth; i++) { @@ -539,10 +536,9 @@ struct dhs_cpack { for (; k < rows; k++) { blockAt[rir] = lhs(k, i).real(); - if (Conjugate) - blockAt[rii] = -lhs(k, i).imag(); - else - blockAt[rii] = lhs(k, i).imag(); + EIGEN_IF_CONSTEXPR(Conjugate) + blockAt[rii] = -lhs(k, i).imag(); + else blockAt[rii] = lhs(k, i).imag(); rir += 1; rii += 1; @@ -563,14 +559,15 @@ struct dhs_pack { for (; i + n * vectorSize <= depth; i += n * vectorSize) { for (Index k = 0; k < n; k++) { - if (UseLhs) { + EIGEN_IF_CONSTEXPR(UseLhs) { bload(block[k], lhs2, 0, i + k * vectorSize); - } else { + } + else { bload(block[k], lhs2, i + k * vectorSize, 0); } } - if (((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { + EIGEN_IF_CONSTEXPR(((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { for (Index k = 0; k < n; k++) { ptranspose(block[k]); } @@ -593,30 +590,31 @@ struct dhs_pack { const DataMapper lhs2 = UseLhs ? lhs.getSubMapper(j, 0) : lhs.getSubMapper(0, j); Index i = 0; - if (PanelMode) ri += vectorSize * offset; + EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * offset; dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize); dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize); dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize); for (; i < depth; i++) { - if (((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { - if (UseLhs) { + EIGEN_IF_CONSTEXPR(((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { + EIGEN_IF_CONSTEXPR(UseLhs) { blockA[ri + 0] = lhs2(0, i); blockA[ri + 1] = lhs2(1, i); blockA[ri + 2] = lhs2(2, i); blockA[ri + 3] = lhs2(3, i); - } else { + } + else { blockA[ri + 0] = lhs2(i, 0); blockA[ri + 1] = lhs2(i, 1); blockA[ri + 2] = lhs2(i, 2); blockA[ri + 3] = lhs2(i, 3); } - } else { + } + else { Packet lhsV; - if (UseLhs) { - lhsV = lhs2.template loadPacket(0, i); - } else { + EIGEN_IF_CONSTEXPR(UseLhs) { lhsV = lhs2.template loadPacket(0, i); } + else { lhsV = lhs2.template loadPacket(i, 0); } pstore(blockA + ri, lhsV); @@ -625,11 +623,11 @@ struct dhs_pack { ri += vectorSize; } - if (PanelMode) ri += vectorSize * (stride - offset - depth); + EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * (stride - offset - depth); } - if (!UseLhs) { - if (PanelMode) ri += offset; + EIGEN_IF_CONSTEXPR(!UseLhs) { + EIGEN_IF_CONSTEXPR(PanelMode) ri += offset; for (; j < rows; j++) { const DataMapper lhs2 = lhs.getSubMapper(0, j); @@ -638,11 +636,12 @@ struct dhs_pack { ri += 1; } - if (PanelMode) ri += stride - depth; + EIGEN_IF_CONSTEXPR(PanelMode) ri += stride - depth; } - } else { + } + else { if (j < rows) { - if (PanelMode) ri += offset * (rows - j); + EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (rows - j); for (Index i = 0; i < depth; i++) { Index k = j; @@ -666,16 +665,17 @@ struct dhs_pack { for (; i + n * vectorSize <= depth; i += n * vectorSize) { for (Index k = 0; k < n; k++) { - if (StorageOrder == RowMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == RowMajor) { block[k].packet[0] = lhs2.template loadPacket(0, i + k * vectorSize); block[k].packet[1] = lhs2.template loadPacket(1, i + k * vectorSize); - } else { + } + else { block[k].packet[0] = lhs2.template loadPacket(0, i + k * vectorSize + 0); block[k].packet[1] = lhs2.template loadPacket(0, i + k * vectorSize + 1); } } - if (StorageOrder == RowMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == RowMajor) { for (Index k = 0; k < n; k++) { ptranspose(block[k]); } @@ -698,17 +698,18 @@ struct dhs_pack { const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - if (PanelMode) ri += vectorSize * offset; + EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * offset; dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize); dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize); dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize); for (; i < depth; i++) { - if (StorageOrder == RowMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == RowMajor) { blockA[ri + 0] = lhs2(0, i); blockA[ri + 1] = lhs2(1, i); - } else { + } + else { Packet2d lhsV = lhs2.template loadPacket(0, i); pstore(blockA + ri, lhsV); } @@ -716,11 +717,11 @@ struct dhs_pack { ri += vectorSize; } - if (PanelMode) ri += vectorSize * (stride - offset - depth); + EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * (stride - offset - depth); } if (j < rows) { - if (PanelMode) ri += offset * (rows - j); + EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (rows - j); for (Index i = 0; i < depth; i++) { Index k = j; @@ -744,12 +745,13 @@ struct dhs_pack { for (; i + n * vectorSize <= depth; i += n * vectorSize) { for (Index k = 0; k < n; k++) { - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { block1[k].packet[0] = rhs2.template loadPacket(i + k * vectorSize, 0); block1[k].packet[1] = rhs2.template loadPacket(i + k * vectorSize, 1); block2[k].packet[0] = rhs2.template loadPacket(i + k * vectorSize, 2); block2[k].packet[1] = rhs2.template loadPacket(i + k * vectorSize, 3); - } else { + } + else { block3[k].packet[0] = rhs2.template loadPacket(i + k * vectorSize + 0, 0); //[a1 a2] block3[k].packet[1] = rhs2.template loadPacket(i + k * vectorSize + 0, 2); //[a3 a4] block3[k].packet[2] = rhs2.template loadPacket(i + k * vectorSize + 1, 0); //[b1 b2] @@ -757,7 +759,7 @@ struct dhs_pack { } } - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { for (Index k = 0; k < n; k++) { ptranspose(block1[k]); ptranspose(block2[k]); @@ -765,12 +767,13 @@ struct dhs_pack { } for (Index k = 0; k < n; k++) { - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { pstore(blockB + ri + k * 4 * vectorSize, block1[k].packet[0]); pstore(blockB + ri + k * 4 * vectorSize + 2, block2[k].packet[0]); pstore(blockB + ri + k * 4 * vectorSize + 4, block1[k].packet[1]); pstore(blockB + ri + k * 4 * vectorSize + 6, block2[k].packet[1]); - } else { + } + else { storeBlock(blockB + ri + k * 4 * vectorSize, block3[k]); } } @@ -788,14 +791,14 @@ struct dhs_pack { const DataMapper rhs2 = rhs.getSubMapper(0, j); Index i = 0; - if (PanelMode) ri += offset * (2 * vectorSize); + EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (2 * vectorSize); dhs_copy<4>(blockB, rhs2, i, ri, depth, vectorSize); dhs_copy<2>(blockB, rhs2, i, ri, depth, vectorSize); dhs_copy<1>(blockB, rhs2, i, ri, depth, vectorSize); for (; i < depth; i++) { - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { blockB[ri + 0] = rhs2(i, 0); blockB[ri + 1] = rhs2(i, 1); @@ -803,7 +806,8 @@ struct dhs_pack { blockB[ri + 0] = rhs2(i, 2); blockB[ri + 1] = rhs2(i, 3); - } else { + } + else { Packet2d rhsV = rhs2.template loadPacket(i, 0); pstore(blockB + ri, rhsV); @@ -815,10 +819,10 @@ struct dhs_pack { ri += vectorSize; } - if (PanelMode) ri += (2 * vectorSize) * (stride - offset - depth); + EIGEN_IF_CONSTEXPR(PanelMode) ri += (2 * vectorSize) * (stride - offset - depth); } - if (PanelMode) ri += offset; + EIGEN_IF_CONSTEXPR(PanelMode) ri += offset; for (; j < cols; j++) { const DataMapper rhs2 = rhs.getSubMapper(0, j); @@ -827,7 +831,7 @@ struct dhs_pack { ri += 1; } - if (PanelMode) ri += stride - depth; + EIGEN_IF_CONSTEXPR(PanelMode) ri += stride - depth; } } }; @@ -844,9 +848,9 @@ struct dhs_pack const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - if (PanelMode) ri += 2 * vectorSize * offset; + EIGEN_IF_CONSTEXPR(PanelMode) ri += 2 * vectorSize * offset; - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { for (; i + 2 <= depth; i += 2) { PacketBlock block; @@ -877,7 +881,8 @@ struct dhs_pack ri += 2 * vectorSize; } - } else { + } + else { for (; i + vectorSize <= depth; i += vectorSize) { PacketBlock block1, block2; @@ -996,15 +1001,15 @@ struct dhs_pack } } - if (PanelMode) ri += 2 * vectorSize * (stride - offset - depth); + EIGEN_IF_CONSTEXPR(PanelMode) ri += 2 * vectorSize * (stride - offset - depth); } for (; j + vectorSize <= rows; j += vectorSize) { const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - if (PanelMode) ri += vectorSize * offset; + EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * offset; - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { for (; i + 2 <= depth; i += 2) { PacketBlock block; @@ -1026,7 +1031,8 @@ struct dhs_pack ri += vectorSize; } - } else { + } + else { for (; i + vectorSize <= depth; i += vectorSize) { PacketBlock block1; @@ -1103,16 +1109,16 @@ struct dhs_pack } } - if (PanelMode) ri += vectorSize * (stride - offset - depth); + EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * (stride - offset - depth); } if (j + 4 <= rows) { const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - if (PanelMode) ri += 4 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) ri += 4 * offset; for (; i + 2 <= depth; i += 2) { - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { PacketBlock block; block.packet[0] = lhs2.template loadPacketPartial(0, i + 0, 4); @@ -1121,7 +1127,8 @@ struct dhs_pack block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val); pstore(blockA + ri, block.packet[0]); - } else { + } + else { blockA[ri + 0] = lhs2(0, i + 0); blockA[ri + 1] = lhs2(0, i + 1); blockA[ri + 2] = lhs2(1, i + 0); @@ -1135,11 +1142,12 @@ struct dhs_pack ri += 2 * 4; } if (depth & 1) { - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { Packet8bf lhsV = lhs2.template loadPacketPartial(0, i + 0, 4); pstore_partial(blockA + ri, lhsV, 4); - } else { + } + else { blockA[ri + 0] = lhs2(0, i); blockA[ri + 1] = lhs2(1, i); blockA[ri + 2] = lhs2(2, i); @@ -1149,12 +1157,12 @@ struct dhs_pack ri += 4; } - if (PanelMode) ri += 4 * (stride - offset - depth); + EIGEN_IF_CONSTEXPR(PanelMode) ri += 4 * (stride - offset - depth); j += 4; } if (j < rows) { - if (PanelMode) ri += offset * (rows - j); + EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (rows - j); Index i = 0; for (; i + 2 <= depth; i += 2) { @@ -1187,10 +1195,10 @@ struct dhs_pack const DataMapper rhs2 = rhs.getSubMapper(0, j); Index i = 0; - if (PanelMode) ri += 4 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) ri += 4 * offset; for (; i + vectorSize <= depth; i += vectorSize) { - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { PacketBlock block; bload(block, rhs2, i, 0); @@ -1223,7 +1231,8 @@ struct dhs_pack #endif storeBlock(blockB + ri, block); - } else { + } + else { PacketBlock block; for (int M = 0; M < 8; M++) { @@ -1245,7 +1254,7 @@ struct dhs_pack ri += 4 * vectorSize; } for (; i + 2 <= depth; i += 2) { - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { blockB[ri + 0] = rhs2(i + 0, 0); blockB[ri + 1] = rhs2(i + 1, 0); blockB[ri + 2] = rhs2(i + 0, 1); @@ -1254,7 +1263,8 @@ struct dhs_pack blockB[ri + 5] = rhs2(i + 1, 2); blockB[ri + 6] = rhs2(i + 0, 3); blockB[ri + 7] = rhs2(i + 1, 3); - } else { + } + else { PacketBlock block; for (int M = 0; M < 2; M++) { @@ -1277,11 +1287,11 @@ struct dhs_pack ri += 4; } - if (PanelMode) ri += 4 * (stride - offset - depth); + EIGEN_IF_CONSTEXPR(PanelMode) ri += 4 * (stride - offset - depth); } if (j < cols) { - if (PanelMode) ri += offset * (cols - j); + EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (cols - j); Index i = 0; for (; i + 2 <= depth; i += 2) { @@ -1311,7 +1321,7 @@ struct dhs_cpack cblock; for (; i + vectorSize <= depth; i += vectorSize) { - if (StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { cblock.packet[0] = lhs2.template loadPacket(0, i + 0); //[a1 a1i] cblock.packet[1] = lhs2.template loadPacket(0, i + 1); //[b1 b1i] @@ -1323,7 +1333,8 @@ struct dhs_cpack(0, i); //[a1 a1i] cblock.packet[1] = lhs2.template loadPacket(1, i); //[a2 a2i] @@ -1337,7 +1348,7 @@ struct dhs_cpack(blockAt + rir, blockr.packet[0]); pstore(blockAt + rii, blocki.packet[0]); @@ -1391,7 +1400,7 @@ struct dhs_cpack EIGEN_ALWAYS_INLINE void pger_common(PacketBlock* acc, const Packet& lhsV, const Packet* rhsV) { - if (NegativeAccumulate) { + EIGEN_IF_CONSTEXPR(NegativeAccumulate) { for (int M = 0; M < N; M++) { acc->packet[M] = vec_nmsub(lhsV, rhsV[M], acc->packet[M]); } - } else { + } + else { for (int M = 0; M < N; M++) { acc->packet[M] = vec_madd(lhsV, rhsV[M], acc->packet[M]); } @@ -1515,14 +1523,16 @@ template * accReal, PacketBlock* accImag, const Packet& lhsV, Packet& lhsVi, const Packet* rhsV, const Packet* rhsVi) { pger_common(accReal, lhsV, rhsV); - if (LhsIsReal) { + EIGEN_IF_CONSTEXPR(LhsIsReal) { pger_common(accImag, lhsV, rhsVi); EIGEN_UNUSED_VARIABLE(lhsVi); - } else { - if (!RhsIsReal) { + } + else { + EIGEN_IF_CONSTEXPR(!RhsIsReal) { pger_common(accReal, lhsVi, rhsVi); pger_common(accImag, lhsV, rhsVi); - } else { + } + else { EIGEN_UNUSED_VARIABLE(rhsVi); } pger_common(accImag, lhsVi, rhsV); @@ -1534,10 +1544,9 @@ EIGEN_ALWAYS_INLINE void pgerc(PacketBlock* accReal, PacketBlock(lhs_ptr); Packet lhsVi; - if (!LhsIsReal) - lhsVi = ploadLhs(lhs_ptr_imag); - else - EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); + EIGEN_IF_CONSTEXPR(!LhsIsReal) + lhsVi = ploadLhs(lhs_ptr_imag); + else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); pgerc_common(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi); } @@ -1575,10 +1584,11 @@ template EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag, const Packet& pMask) { - if (mask && (sizeof(__UNPACK_TYPE__(Packet)) == sizeof(float))) { + EIGEN_IF_CONSTEXPR(mask && (sizeof(__UNPACK_TYPE__(Packet)) == sizeof(float))) { band(aReal, pMask); band(aImag, pMask); - } else { + } + else { EIGEN_UNUSED_VARIABLE(pMask); } @@ -1597,20 +1607,21 @@ EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col) { - if (StorageOrder == RowMajor) { + EIGEN_IF_CONSTEXPR(StorageOrder == RowMajor) { for (int M = 0; M < N; M++) { acc.packet[M] = res.template loadPacket(row + M, col); } - if (Complex) { + EIGEN_IF_CONSTEXPR(Complex) { for (int M = 0; M < N; M++) { acc.packet[M + N] = res.template loadPacket(row + M, col + accCols); } } - } else { + } + else { for (int M = 0; M < N; M++) { acc.packet[M] = res.template loadPacket(row, col + M); } - if (Complex && full) { + EIGEN_IF_CONSTEXPR(Complex && full) { for (int M = 0; M < N; M++) { acc.packet[M + N] = res.template loadPacket(row + accCols, col + M); } @@ -1632,7 +1643,7 @@ EIGEN_ALWAYS_INLINE void bload_partial(PacketBlock& for (Index M = 0; M < N; M++) { acc.packet[M] = res.template loadPacketPartial(row, M, elements); } - if (Complex && full) { + EIGEN_IF_CONSTEXPR(Complex && full) { for (Index M = 0; M < N; M++) { acc.packet[M + N] = res.template loadPacketPartial(row + accCols, M, elements); } @@ -1696,9 +1707,8 @@ EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha, const Packet& pMask) { - if (mask) { - band(accZ, pMask); - } else { + EIGEN_IF_CONSTEXPR(mask) { band(accZ, pMask); } + else { EIGEN_UNUSED_VARIABLE(pMask); } @@ -1710,22 +1720,21 @@ EIGEN_ALWAYS_INLINE void pbroadcastN(const __UNPACK_TYPE__(Packet) * ap0, const const __UNPACK_TYPE__(Packet) * ap2, Packet& a0, Packet& a1, Packet& a2, Packet& a3) { a0 = pset1(ap0[0]); - if (N == 4) { + EIGEN_IF_CONSTEXPR(N == 4) { a1 = pset1(ap0[1]); a2 = pset1(ap0[2]); a3 = pset1(ap0[3]); EIGEN_UNUSED_VARIABLE(ap1); EIGEN_UNUSED_VARIABLE(ap2); - } else { - if (N > 1) { - a1 = pset1(ap1[0]); - } else { + } + else { + EIGEN_IF_CONSTEXPR(N > 1) { a1 = pset1(ap1[0]); } + else { EIGEN_UNUSED_VARIABLE(a1); EIGEN_UNUSED_VARIABLE(ap1); } - if (N > 2) { - a2 = pset1(ap2[0]); - } else { + EIGEN_IF_CONSTEXPR(N > 2) { a2 = pset1(ap2[0]); } + else { EIGEN_UNUSED_VARIABLE(a2); EIGEN_UNUSED_VARIABLE(ap2); } @@ -1763,7 +1772,7 @@ EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock& taccReal, Packet acc1.packet[M].v = vec_mergeh(taccReal.packet[M], taccImag.packet[M]); } - if (full) { + EIGEN_IF_CONSTEXPR(full) { for (int M = 0; M < N; M++) { acc2.packet[M].v = vec_mergel(taccReal.packet[M], taccImag.packet[M]); } @@ -1780,7 +1789,7 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

(tRes.packet[M], acc1.packet[M]); } - if (full) { + EIGEN_IF_CONSTEXPR(full) { for (int M = 0; M < N; M++) { acc2.packet[M] = padd(tRes.packet[M + N], acc2.packet[M]); } @@ -1799,31 +1808,28 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

peel) && (peel != 0)) { \ - bsetzero(accZero##peel); \ - } else { \ - EIGEN_UNUSED_VARIABLE(accZero##peel); \ +#define MICRO_ZERO_PEEL(peel) \ + EIGEN_IF_CONSTEXPR((PEEL_ROW > peel) && (peel != 0)) { bsetzero(accZero##peel); } \ + else { \ + EIGEN_UNUSED_VARIABLE(accZero##peel); \ } -#define MICRO_ADD(ptr, N) \ - if (MICRO_NORMAL_ROWS) { \ - MICRO_RHS(ptr, 0) += (accRows * N); \ - } else { \ - MICRO_RHS(ptr, 0) += N; \ - MICRO_RHS(ptr, 1) += N; \ - if (accRows == 3) { \ - MICRO_RHS(ptr, 2) += N; \ - } \ +#define MICRO_ADD(ptr, N) \ + EIGEN_IF_CONSTEXPR(MICRO_NORMAL_ROWS) { MICRO_RHS(ptr, 0) += (accRows * N); } \ + else { \ + MICRO_RHS(ptr, 0) += N; \ + MICRO_RHS(ptr, 1) += N; \ + EIGEN_IF_CONSTEXPR(accRows == 3) { MICRO_RHS(ptr, 2) += N; } \ } #define MICRO_ADD_ROWS(N) MICRO_ADD(ptr, N) #define MICRO_BROADCAST1(peel, ptr, rhsV, real) \ - if (MICRO_NORMAL_ROWS) { \ + EIGEN_IF_CONSTEXPR(MICRO_NORMAL_ROWS) { \ pbroadcastN(MICRO_RHS(ptr, 0) + (accRows * peel), MICRO_RHS(ptr, 0), MICRO_RHS(ptr, 0), \ rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ - } else { \ + } \ + else { \ pbroadcastN(MICRO_RHS(ptr, 0) + peel, MICRO_RHS(ptr, 1) + peel, MICRO_RHS(ptr, 2) + peel, \ rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ } @@ -1839,18 +1845,18 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

& taccReal, PacketBlock

peel) { \ + EIGEN_IF_CONSTEXPR(PEEL_ROW > peel) { \ MICRO_BROADCAST(peel) \ pger(&accZero##peel, lhs_ptr + (remaining_rows * peel), rhsV##peel); \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ } @@ -1872,7 +1879,7 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

peel) { \ + EIGEN_IF_CONSTEXPR(PEEL_ROW > peel) { \ for (Index i = 0; i < accRows; i++) { \ accZero##sum.packet[i] += accZero##peel.packet[i]; \ } \ @@ -1883,22 +1890,18 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

EIGEN_ALWAYS_INLINE void MICRO_EXTRA_ROW(const Scalar*& lhs_ptr, const Scalar*& rhs_ptr0, const Scalar*& rhs_ptr1, @@ -1958,21 +1961,17 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_row_iteration(const DataMapper& res, cons #endif } -#define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col) \ - switch (value) { \ - default: \ - MICRO_EXTRA_UNROLL(1) \ - break; \ - case 2: \ - if (is_col || (sizeof(Scalar) == sizeof(float))) { \ - MICRO_EXTRA_UNROLL(2) \ - } \ - break; \ - case 3: \ - if (is_col || (sizeof(Scalar) == sizeof(float))) { \ - MICRO_EXTRA_UNROLL(3) \ - } \ - break; \ +#define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col) \ + switch (value) { \ + default: \ + MICRO_EXTRA_UNROLL(1) \ + break; \ + case 2: \ + EIGEN_IF_CONSTEXPR(is_col || (sizeof(Scalar) == sizeof(float))) { MICRO_EXTRA_UNROLL(2) } \ + break; \ + case 3: \ + EIGEN_IF_CONSTEXPR(is_col || (sizeof(Scalar) == sizeof(float))) { MICRO_EXTRA_UNROLL(3) } \ + break; \ } #define MICRO_EXTRA_ROWS(N) \ @@ -1991,16 +1990,17 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs func(0, peel) func(1, peel) func(2, peel) func(3, peel) func(4, peel) func(5, peel) func(6, peel) func(7, peel) #define MICRO_WORK_ONE(iter, peel) \ - if (unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ pger_common(&accZero##iter, lhsV##iter, rhsV##peel); \ } #define MICRO_TYPE_PEEL4(func, func2, peel) \ - if (PEEL > peel) { \ + EIGEN_IF_CONSTEXPR(PEEL > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \ MICRO_BROADCAST(peel) \ MICRO_UNROLL_WORK(func, func2, peel) \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ } @@ -2021,11 +2021,10 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs #define MICRO_ONE4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_ONE, 1) -#define MICRO_DST_PTR_ONE(iter) \ - if (unroll_factor > iter) { \ - bsetzero(accZero##iter); \ - } else { \ - EIGEN_UNUSED_VARIABLE(accZero##iter); \ +#define MICRO_DST_PTR_ONE(iter) \ + EIGEN_IF_CONSTEXPR(unroll_factor > iter) { bsetzero(accZero##iter); } \ + else { \ + EIGEN_UNUSED_VARIABLE(accZero##iter); \ } #define MICRO_DST_PTR MICRO_UNROLL(MICRO_DST_PTR_ONE) @@ -2036,12 +2035,13 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs #ifdef USE_PARTIAL_PACKETS #define MICRO_STORE_ONE(iter) \ - if (unroll_factor > iter) { \ - if (MICRO_NORMAL_PARTIAL(iter)) { \ + EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR(MICRO_NORMAL_PARTIAL(iter)) { \ bload(acc, res, row + iter * accCols, 0); \ bscale(acc, accZero##iter, pAlpha); \ bstore(acc, res, row + iter * accCols); \ - } else { \ + } \ + else { \ bload_partial(acc, res, row + iter * accCols, accCols2); \ bscale(acc, accZero##iter, pAlpha); \ bstore_partial(acc, res, row + iter * accCols, accCols2); \ @@ -2049,7 +2049,7 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs } #else #define MICRO_STORE_ONE(iter) \ - if (unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ bload(acc, res, row + iter * accCols, 0); \ bscale(acc, accZero##iter, pAlpha, pMask); \ bstore(acc, res, row + iter * accCols); \ @@ -2221,49 +2221,48 @@ EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const #define MICRO_COMPLEX_UNROLL(func) func(0) func(1) func(2) func(3) -#define MICRO_COMPLEX_ZERO_PEEL(peel) \ - if ((PEEL_COMPLEX_ROW > peel) && (peel != 0)) { \ - bsetzero(accReal##peel); \ - bsetzero(accImag##peel); \ - } else { \ - EIGEN_UNUSED_VARIABLE(accReal##peel); \ - EIGEN_UNUSED_VARIABLE(accImag##peel); \ +#define MICRO_COMPLEX_ZERO_PEEL(peel) \ + EIGEN_IF_CONSTEXPR((PEEL_COMPLEX_ROW > peel) && (peel != 0)) { \ + bsetzero(accReal##peel); \ + bsetzero(accImag##peel); \ + } \ + else { \ + EIGEN_UNUSED_VARIABLE(accReal##peel); \ + EIGEN_UNUSED_VARIABLE(accImag##peel); \ } -#define MICRO_COMPLEX_ADD_ROWS(N, used) \ - MICRO_ADD(ptr_real, N) \ - if (!RhsIsReal) { \ - MICRO_ADD(ptr_imag, N) \ - } else if (used) { \ - EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \ - EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \ - EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \ +#define MICRO_COMPLEX_ADD_ROWS(N, used) \ + MICRO_ADD(ptr_real, N) \ + EIGEN_IF_CONSTEXPR(!RhsIsReal) { MICRO_ADD(ptr_imag, N) } \ + else if (used) { \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \ } -#define MICRO_COMPLEX_BROADCAST(peel) \ - MICRO_BROADCAST1(peel, ptr_real, rhsV, false) \ - if (!RhsIsReal) { \ - MICRO_BROADCAST1(peel, ptr_imag, rhsVi, false) \ - } else { \ - EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ +#define MICRO_COMPLEX_BROADCAST(peel) \ + MICRO_BROADCAST1(peel, ptr_real, rhsV, false) \ + EIGEN_IF_CONSTEXPR(!RhsIsReal) { MICRO_BROADCAST1(peel, ptr_imag, rhsVi, false) } \ + else { \ + EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } -#define MICRO_COMPLEX_BROADCAST_EXTRA \ - Packet rhsV[4], rhsVi[4]; \ - MICRO_BROADCAST_EXTRA1(ptr_real, rhsV, false) \ - if (!RhsIsReal) { \ - MICRO_BROADCAST_EXTRA1(ptr_imag, rhsVi, false) \ - } else { \ - EIGEN_UNUSED_VARIABLE(rhsVi); \ - } \ +#define MICRO_COMPLEX_BROADCAST_EXTRA \ + Packet rhsV[4], rhsVi[4]; \ + MICRO_BROADCAST_EXTRA1(ptr_real, rhsV, false) \ + EIGEN_IF_CONSTEXPR(!RhsIsReal) { MICRO_BROADCAST_EXTRA1(ptr_imag, rhsVi, false) } \ + else { \ + EIGEN_UNUSED_VARIABLE(rhsVi); \ + } \ MICRO_COMPLEX_ADD_ROWS(1, true) #define MICRO_COMPLEX_SRC2_PTR \ MICRO_SRC2(ptr_real, strideB* advanceCols, 0) \ - if (!RhsIsReal) { \ + EIGEN_IF_CONSTEXPR(!RhsIsReal) { \ MICRO_RHS(ptr_imag, 0) = rhs_base + MICRO_NEW_ROWS * strideB; \ MICRO_SRC2(ptr_imag, strideB* advanceCols, strideB) \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \ EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \ EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \ @@ -2272,22 +2271,22 @@ EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const #define MICRO_COMPLEX_ZERO_PEEL_ROW MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_ZERO_PEEL) #define MICRO_COMPLEX_WORK_PEEL(peel) \ - if (PEEL_COMPLEX_ROW > peel) { \ + EIGEN_IF_CONSTEXPR(PEEL_COMPLEX_ROW > peel) { \ MICRO_COMPLEX_BROADCAST(peel) \ pgerc( \ &accReal##peel, &accImag##peel, lhs_ptr_real + (remaining_rows * peel), \ lhs_ptr_imag + (remaining_rows * peel), rhsV##peel, rhsVi##peel); \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } -#define MICRO_COMPLEX_ADD_COLS(size) \ - lhs_ptr_real += (remaining_rows * size); \ - if (!LhsIsReal) \ - lhs_ptr_imag += (remaining_rows * size); \ - else \ - EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); +#define MICRO_COMPLEX_ADD_COLS(size) \ + lhs_ptr_real += (remaining_rows * size); \ + EIGEN_IF_CONSTEXPR(!LhsIsReal) \ + lhs_ptr_imag += (remaining_rows * size); \ + else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); #define MICRO_COMPLEX_WORK_PEEL_ROW \ Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4]; \ @@ -2297,7 +2296,7 @@ EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const MICRO_COMPLEX_ADD_ROWS(PEEL_COMPLEX_ROW, false) #define MICRO_COMPLEX_ADD_PEEL(peel, sum) \ - if (PEEL_COMPLEX_ROW > peel) { \ + EIGEN_IF_CONSTEXPR(PEEL_COMPLEX_ROW > peel) { \ for (Index i = 0; i < accRows; i++) { \ accReal##sum.packet[i] += accReal##peel.packet[i]; \ accImag##sum.packet[i] += accImag##peel.packet[i]; \ @@ -2333,10 +2332,9 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper& r const Scalar *rhs_ptr_imag0 = NULL, *rhs_ptr_imag1 = NULL, *rhs_ptr_imag2 = NULL; const Scalar* lhs_ptr_real = lhs_base + advanceRows * row * strideA + remaining_rows * offsetA; const Scalar* lhs_ptr_imag = NULL; - if (!LhsIsReal) - lhs_ptr_imag = lhs_ptr_real + remaining_rows * strideA; - else - EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); + EIGEN_IF_CONSTEXPR(!LhsIsReal) + lhs_ptr_imag = lhs_ptr_real + remaining_rows * strideA; + else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); PacketBlock accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3; PacketBlock taccReal, taccImag; PacketBlock acc0, acc1; @@ -2354,9 +2352,7 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper& r do { MICRO_COMPLEX_PREFETCHN(accRows) EIGEN_POWER_PREFETCH(lhs_ptr_real); - if (!LhsIsReal) { - EIGEN_POWER_PREFETCH(lhs_ptr_imag); - } + EIGEN_IF_CONSTEXPR(!LhsIsReal) { EIGEN_POWER_PREFETCH(lhs_ptr_imag); } MICRO_COMPLEX_WORK_PEEL_ROW } while ((k += PEEL_COMPLEX_ROW) + PEEL_COMPLEX_ROW <= remaining_depth); MICRO_COMPLEX_ADD_PEEL_ROW @@ -2373,9 +2369,7 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper& r bscalec(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); bcouple(taccReal, taccImag, tRes, acc0, acc1); bstore(acc0, res, row + 0); - if (full) { - bstore(acc1, res, row + accColsC); - } + EIGEN_IF_CONSTEXPR(full) { bstore(acc1, res, row + accColsC); } } else { bscalec(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); bcouple(taccReal, taccImag, tRes, acc0, acc1); @@ -2386,7 +2380,7 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper& r } } else { bstore(acc0, res, row + 0); - if (full) { + EIGEN_IF_CONSTEXPR(full) { for (Index j = 0; j < accRows; j++) { res(row + accColsC, j) = pfirst(acc1.packet[j]); } @@ -2414,18 +2408,19 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(const DataMapper& res, const Sca func(0, peel) func(1, peel) func(2, peel) func(3, peel) #define MICRO_COMPLEX_WORK_ONE4(iter, peel) \ - if (unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ pgerc_common( \ &accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \ } #define MICRO_COMPLEX_TYPE_PEEL4(func, func2, peel) \ - if (PEEL_COMPLEX > peel) { \ + EIGEN_IF_CONSTEXPR(PEEL_COMPLEX > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3; \ Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3; \ MICRO_COMPLEX_BROADCAST(peel) \ MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } @@ -2448,10 +2443,11 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(const DataMapper& res, const Sca #define MICRO_COMPLEX_ONE4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1) #define MICRO_COMPLEX_DST_PTR_ONE(iter) \ - if (unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ bsetzero(accReal##iter); \ bsetzero(accImag##iter); \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(accReal##iter); \ EIGEN_UNUSED_VARIABLE(accImag##iter); \ } @@ -2463,16 +2459,14 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(const DataMapper& res, const Sca #define MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_PREFETCH_ONE) #define MICRO_COMPLEX_STORE_ONE(iter) \ - if (unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ constexpr bool full = ((MICRO_NORMAL(iter)) || (accCols2 > accColsC)); \ bload(tRes, res, row + iter * accCols, 0); \ bscalec(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, \ taccImag, pMask); \ bcouple(taccReal, taccImag, tRes, acc0, acc1); \ bstore(acc0, res, row + iter * accCols + 0); \ - if (full) { \ - bstore(acc1, res, row + iter * accCols + accColsC); \ - } \ + EIGEN_IF_CONSTEXPR(full) { bstore(acc1, res, row + iter * accCols + accColsC); } \ } #define MICRO_COMPLEX_STORE MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_STORE_ONE) @@ -2638,9 +2632,8 @@ EIGEN_ALWAYS_INLINE Packet4f loadAndMultiplyF32(Packet4f acc, const Packet4f pAl template EIGEN_ALWAYS_INLINE void storeF32(float*& result, Packet4f result_block, Index rows, Index extra_rows) { - if (lhsExtraRows) { - pstoreu_partial(result, result_block, extra_rows); - } else { + EIGEN_IF_CONSTEXPR(lhsExtraRows) { pstoreu_partial(result, result_block, extra_rows); } + else { pstoreu(result, result_block); } result += rows; @@ -2650,12 +2643,13 @@ template EIGEN_ALWAYS_INLINE void storeResults(Packet4f (&acc)[4], Index rows, const Packet4f pAlpha, float* result, Index extra_cols, Index extra_rows) { Index x = 0; - if (rhsExtraCols) { + EIGEN_IF_CONSTEXPR(rhsExtraCols) { do { Packet4f result_block = loadAndMultiplyF32(acc[x], pAlpha, result); storeF32(result, result_block, rows, extra_rows); } while (++x < extra_cols); - } else { + } + else { Packet4f result_block[4]; float* result2 = result; do { @@ -2689,23 +2683,18 @@ EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Lo(Packet8us data) { template EIGEN_ALWAYS_INLINE void storeConvertTwoBF16(float* to, PacketBlock& block, Index extra = 0) { - if (N < 4) { - pstoreu_partial(to + 0, oneConvertBF16Hi(block.packet[0].m_val), extra); - } else if (N >= (M * 8 + 4)) { + EIGEN_IF_CONSTEXPR(N < 4) { pstoreu_partial(to + 0, oneConvertBF16Hi(block.packet[0].m_val), extra); } + else EIGEN_IF_CONSTEXPR(N >= (M * 8 + 4)) { pstoreu(to + 0, oneConvertBF16Hi(block.packet[M].m_val)); - if (N >= 8) { - pstoreu(to + 4, oneConvertBF16Lo(block.packet[M].m_val)); - } + EIGEN_IF_CONSTEXPR(N >= 8) { pstoreu(to + 4, oneConvertBF16Lo(block.packet[M].m_val)); } } } template EIGEN_ALWAYS_INLINE void storeConvertBlockBF16(float* to, PacketBlock& block, Index extra) { storeConvertTwoBF16(to + 0, block, extra); - if (N >= 16) { - storeConvertTwoBF16(to + 8, block); - } - if (N >= 32) { + EIGEN_IF_CONSTEXPR(N >= 16) { storeConvertTwoBF16(to + 8, block); } + EIGEN_IF_CONSTEXPR(N >= 32) { storeConvertTwoBF16(to + 16, block); storeConvertTwoBF16(to + 24, block); } @@ -2713,9 +2702,8 @@ EIGEN_ALWAYS_INLINE void storeConvertBlockBF16(float* to, PacketBlock EIGEN_ALWAYS_INLINE Packet8bf loadBF16fromResult(bfloat16* src, Index resInc) { - if (non_unit_stride) { - return pgather(src + delta * resInc, resInc); - } else { + EIGEN_IF_CONSTEXPR(non_unit_stride) { return pgather(src + delta * resInc, resInc); } + else { return ploadu(src + delta); } } @@ -2757,7 +2745,7 @@ EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32DupOne(float* result, Index } for (Index j = 0; j < 4 * size; j += 4) { - if (lhsExtraRows) { + EIGEN_IF_CONSTEXPR(lhsExtraRows) { Packet4f z = pset1(float(0)); Index i = 0; do { @@ -2766,7 +2754,8 @@ EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32DupOne(float* result, Index do { pstoreu(result + (j + i) * 4, z); } while (++i < 4); - } else { + } + else { for (Index i = 0; i < 4; i++) { pstoreu(result + (j + i) * 4, dup[j + i]); } @@ -2796,17 +2785,15 @@ EIGEN_ALWAYS_INLINE void convertPointerBF16toF32(Index& i, float* result, Index while (i + size <= rows) { PacketBlock r32; r32.packet[0] = loadBF16fromResult(src, resInc); - if (size >= 16) { - r32.packet[1] = loadBF16fromResult(src, resInc); - } - if (size >= 32) { + EIGEN_IF_CONSTEXPR(size >= 16) { r32.packet[1] = loadBF16fromResult(src, resInc); } + EIGEN_IF_CONSTEXPR(size >= 32) { r32.packet[2] = loadBF16fromResult(src, resInc); r32.packet[3] = loadBF16fromResult(src, resInc); } storeConvertBlockBF16(result + i, r32, rows & 3); i += extra; src += extra * resInc; - if (size != 32) break; + EIGEN_IF_CONSTEXPR(size != 32) break; } } @@ -2872,7 +2859,7 @@ EIGEN_ALWAYS_INLINE void outputResultsVSX(Packet4f (&acc)[num_acc][4], Index row for (Index i = 0; i < real_rhs; i++, result += 4 * rows, k++) { storeResults(acc[k], rows, pAlpha, result, extra_cols, extra_rows); } - if (rhsExtraCols) { + EIGEN_IF_CONSTEXPR(rhsExtraCols) { storeResults(acc[k], rows, pAlpha, result, extra_cols, extra_rows); } } @@ -2880,11 +2867,12 @@ EIGEN_ALWAYS_INLINE void outputResultsVSX(Packet4f (&acc)[num_acc][4], Index row template EIGEN_ALWAYS_INLINE void loadTwoRhsFloat32(const float* block, Index strideB, Index i, Packet4f& dhs0, Packet4f& dhs1) { dhs0 = ploadu(block + strideB * i + 0); - if (zero) { + EIGEN_IF_CONSTEXPR(zero) { Packet4f dhs2 = pset1(float(0)); dhs1 = vec_mergel(dhs0, dhs2); dhs0 = vec_mergeh(dhs0, dhs2); - } else { + } + else { dhs1 = ploadu(block + strideB * i + 4); } } @@ -2899,7 +2887,7 @@ EIGEN_ALWAYS_INLINE void KLoop(const float* indexA, const float* indexB, Packet4 for (Index i = 0; i < real_rhs; i += 2) { loadTwoRhsFloat32(indexB + k * 4, strideB, i, rhs[i + 0], rhs[i + 1]); } - if (rhsExtraCols) { + EIGEN_IF_CONSTEXPR(rhsExtraCols) { loadTwoRhsFloat32(indexB + k * extra_cols - offsetB, strideB, real_rhs, rhs[real_rhs + 0], rhs[real_rhs + 1]); } @@ -2960,7 +2948,7 @@ template EIGEN_ALWAYS_INLINE void colVSXLoopBodyExtraN(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float* indexA, const float* blockB, Index strideB, Index offsetB, float* result) { - if (MAX_BFLOAT16_ACC_VSX > num_acc) { + EIGEN_IF_CONSTEXPR(MAX_BFLOAT16_ACC_VSX > num_acc) { colVSXLoopBody(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); } @@ -2983,7 +2971,7 @@ void colVSXLoopBodyExtra(Index col, Index depth, Index cols, Index rows, const P offsetB, result); break; default: - if (rhsExtraCols) { + EIGEN_IF_CONSTEXPR(rhsExtraCols) { colVSXLoopBody<1, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); } break; @@ -3037,16 +3025,14 @@ EIGEN_ALWAYS_INLINE void convertBF16toF32(Index& i, float* result, Index rows, c while (i + size <= rows) { PacketBlock r32; r32.packet[0] = src.template loadPacket(i + 0); - if (size >= 16) { - r32.packet[1] = src.template loadPacket(i + 8); - } - if (size >= 32) { + EIGEN_IF_CONSTEXPR(size >= 16) { r32.packet[1] = src.template loadPacket(i + 8); } + EIGEN_IF_CONSTEXPR(size >= 32) { r32.packet[2] = src.template loadPacket(i + 16); r32.packet[3] = src.template loadPacket(i + 24); } storeConvertBlockBF16(result + i, r32, rows & 3); i += extra; - if (size != 32) break; + EIGEN_IF_CONSTEXPR(size != 32) break; } } diff --git a/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc b/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc index fbf7eab04..b3a90c640 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc +++ b/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc @@ -92,23 +92,17 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal func(6, N, which) func(7, N, which) #define GEMV_UNUSED_VAR(iter, N, which) \ - if (GEMV_GETN(N) <= iter) { \ - EIGEN_UNUSED_VARIABLE(which##iter); \ - } + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) <= iter) { EIGEN_UNUSED_VARIABLE(which##iter); } #define GEMV_UNUSED_EXTRA_VAR(iter, N, which) \ - if (N <= iter) { \ - EIGEN_UNUSED_VARIABLE(which##iter); \ - } + EIGEN_IF_CONSTEXPR(N <= iter) { EIGEN_UNUSED_VARIABLE(which##iter); } #define GEMV_UNUSED_EXTRA(N, which) GEMV_UNROLL3(GEMV_UNUSED_EXTRA_VAR, N, which) #define GEMV_UNUSED(N, which) GEMV_UNROLL3(GEMV_UNUSED_VAR, N, which) -#define GEMV_INIT_MMA(iter, N) \ - if (GEMV_GETN(N) > iter) { \ - __builtin_mma_xxsetaccz(&e##iter); \ - } +#define GEMV_INIT_MMA(iter, N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { __builtin_mma_xxsetaccz(&e##iter); } #if EIGEN_COMP_LLVM #define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \ @@ -120,38 +114,41 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal #endif #define GEMV_LOAD1A_COL_MMA(iter, N) \ - if (GEMV_GETN(N) > iter) { \ - if (GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ g##iter = GEMV_LOADPACKET_COL(iter); \ EIGEN_UNUSED_VARIABLE(b##iter); \ - } else { \ + } \ + else { \ GEMV_LOADPAIR_COL_MMA(iter, iter << 1) \ EIGEN_UNUSED_VARIABLE(g##iter); \ } \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(b##iter); \ EIGEN_UNUSED_VARIABLE(g##iter); \ } -#define GEMV_WORK1A_COL_MMA(iter, N) \ - if (GEMV_GETN(N) > iter) { \ - if (GEMV_IS_FLOAT) { \ - pger_vecMMA_acc(&e##iter, a0, g##iter); \ - } else { \ - pger_vecMMA_acc(&e##iter, b##iter, a0); \ - } \ +#define GEMV_WORK1A_COL_MMA(iter, N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { pger_vecMMA_acc(&e##iter, a0, g##iter); } \ + else { \ + pger_vecMMA_acc(&e##iter, b##iter, a0); \ + } \ } #define GEMV_LOAD1B_COL_MMA(iter1, iter2, iter3, N) \ - if (GEMV_GETN(N) > iter1) { \ - if (GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ GEMV_LOADPAIR_COL_MMA(iter2, iter2) \ EIGEN_UNUSED_VARIABLE(b##iter3); \ - } else { \ + } \ + else { \ GEMV_LOADPAIR_COL_MMA(iter2, iter2 << 1) \ GEMV_LOADPAIR_COL_MMA(iter3, iter3 << 1) \ } \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(b##iter2); \ EIGEN_UNUSED_VARIABLE(b##iter3); \ } \ @@ -159,31 +156,30 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal EIGEN_UNUSED_VARIABLE(g##iter3); #define GEMV_WORK1B_COL_MMA(iter1, iter2, iter3, N) \ - if (GEMV_GETN(N) > iter1) { \ - if (GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ LhsPacket h[2]; \ __builtin_vsx_disassemble_pair(reinterpret_cast(h), &b##iter2); \ pger_vecMMA_acc(&e##iter2, a0, h[0]); \ pger_vecMMA_acc(&e##iter3, a0, h[1]); \ - } else { \ + } \ + else { \ pger_vecMMA_acc(&e##iter2, b##iter2, a0); \ pger_vecMMA_acc(&e##iter3, b##iter3, a0); \ } \ } #if EIGEN_COMP_LLVM -#define GEMV_LOAD_COL_MMA(N) \ - if (GEMV_GETN(N) > 1) { \ - GEMV_UNROLL_HALF(GEMV_LOAD1B_COL_MMA, (N >> 1)) \ - } else { \ - GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) \ +#define GEMV_LOAD_COL_MMA(N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > 1) { GEMV_UNROLL_HALF(GEMV_LOAD1B_COL_MMA, (N >> 1)) } \ + else { \ + GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) \ } -#define GEMV_WORK_COL_MMA(N) \ - if (GEMV_GETN(N) > 1) { \ - GEMV_UNROLL_HALF(GEMV_WORK1B_COL_MMA, (N >> 1)) \ - } else { \ - GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) \ +#define GEMV_WORK_COL_MMA(N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > 1) { GEMV_UNROLL_HALF(GEMV_WORK1B_COL_MMA, (N >> 1)) } \ + else { \ + GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) \ } #else #define GEMV_LOAD_COL_MMA(N) GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) @@ -192,9 +188,9 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal #endif #define GEMV_DISASSEMBLE_MMA(iter, N) \ - if (GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ __builtin_mma_disassemble_acc(&result##iter.packet, &e##iter); \ - if (!GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR(!GEMV_IS_FLOAT) { \ result##iter.packet[0][1] = result##iter.packet[1][0]; \ result##iter.packet[2][1] = result##iter.packet[3][0]; \ } \ @@ -204,15 +200,17 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal b##iter1 = *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize)); #define GEMV_LOAD2_COL_MMA(iter1, iter2, iter3, N) \ - if (GEMV_GETN(N) > iter1) { \ - if (GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ GEMV_LOADPAIR2_COL_MMA(iter2, iter2); \ EIGEN_UNUSED_VARIABLE(b##iter3); \ - } else { \ + } \ + else { \ GEMV_LOADPAIR2_COL_MMA(iter2, iter2 << 1); \ GEMV_LOADPAIR2_COL_MMA(iter3, iter3 << 1); \ } \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(b##iter2); \ EIGEN_UNUSED_VARIABLE(b##iter3); \ } @@ -226,96 +224,88 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal GEMV_BUILDPAIR_MMA(b##iter2, f##iter2[0], f##iter2[1]); #else #define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \ - if (GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ __asm__("xvmaddasp %0,%x1,%x3\n\txvmaddasp %L0,%x2,%x3" \ : "+&d"(b##iter2) \ : "wa"(result##iter3.packet[0]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \ - } else { \ + } \ + else { \ __asm__("xvmaddadp %0,%x1,%x3\n\txvmaddadp %L0,%x2,%x3" \ : "+&d"(b##iter2) \ : "wa"(result##iter2.packet[2]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \ } #endif -#define GEMV_WORK2_COL_MMA(iter1, iter2, iter3, N) \ - if (GEMV_GETN(N) > iter1) { \ - if (GEMV_IS_FLOAT) { \ - GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter2); \ - } else { \ - GEMV_WORKPAIR2_COL_MMA(iter2, iter2, iter2 << 1); \ - GEMV_WORKPAIR2_COL_MMA(iter3, iter3, iter3 << 1); \ - } \ +#define GEMV_WORK2_COL_MMA(iter1, iter2, iter3, N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter2); } \ + else { \ + GEMV_WORKPAIR2_COL_MMA(iter2, iter2, iter2 << 1); \ + GEMV_WORKPAIR2_COL_MMA(iter3, iter3, iter3 << 1); \ + } \ } #define GEMV_STOREPAIR2_COL_MMA(iter1, iter2) \ *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize)) = b##iter1; #define GEMV_STORE_COL_MMA(iter, N) \ - if (GEMV_GETN(N) > iter) { \ - if (GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ storeMaddData(res + i + (iter * ResPacketSize), palpha, result##iter.packet[0]); \ - } else { \ + } \ + else { \ GEMV_LOADPAIR2_COL_MMA(iter, iter << 1) \ GEMV_WORKPAIR2_COL_MMA(iter, iter, iter << 1) \ GEMV_STOREPAIR2_COL_MMA(iter, iter << 1) \ } \ } -#define GEMV_STORE2_COL_MMA(iter1, iter2, iter3, N) \ - if (GEMV_GETN(N) > iter1) { \ - if (GEMV_IS_FLOAT) { \ - GEMV_STOREPAIR2_COL_MMA(iter2, iter2); \ - } else { \ - GEMV_STOREPAIR2_COL_MMA(iter2, iter2 << 1) \ - GEMV_STOREPAIR2_COL_MMA(iter3, iter3 << 1) \ - } \ +#define GEMV_STORE2_COL_MMA(iter1, iter2, iter3, N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { GEMV_STOREPAIR2_COL_MMA(iter2, iter2); } \ + else { \ + GEMV_STOREPAIR2_COL_MMA(iter2, iter2 << 1) \ + GEMV_STOREPAIR2_COL_MMA(iter3, iter3 << 1) \ + } \ } -#define GEMV_PROCESS_COL_ONE_MMA(N) \ - GEMV_UNROLL(GEMV_INIT_MMA, N) \ - Index j = j2; \ - __vector_pair b0, b1, b2, b3, b4, b5, b6, b7; \ - do { \ - LhsPacket g0, g1, g2, g3, g4, g5, g6, g7; \ - RhsPacket a0 = pset1(rhs2(j, 0)); \ - GEMV_UNROLL(GEMV_PREFETCH, N) \ - GEMV_LOAD_COL_MMA(N) \ - GEMV_WORK_COL_MMA(N) \ - } while (++j < jend); \ - GEMV_UNROLL(GEMV_DISASSEMBLE_MMA, N) \ - if (GEMV_GETN(N) <= 1) { \ - GEMV_UNROLL(GEMV_STORE_COL_MMA, N) \ - } else { \ - GEMV_UNROLL_HALF(GEMV_LOAD2_COL_MMA, (N >> 1)) \ - GEMV_UNROLL_HALF(GEMV_WORK2_COL_MMA, (N >> 1)) \ - GEMV_UNROLL_HALF(GEMV_STORE2_COL_MMA, (N >> 1)) \ - } \ - i += (ResPacketSize * N); +#define GEMV_PROCESS_COL_ONE_MMA(N) \ + GEMV_UNROLL(GEMV_INIT_MMA, N) \ + Index j = j2; \ + __vector_pair b0, b1, b2, b3, b4, b5, b6, b7; \ + do { \ + LhsPacket g0, g1, g2, g3, g4, g5, g6, g7; \ + RhsPacket a0 = pset1(rhs2(j, 0)); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_LOAD_COL_MMA(N) \ + GEMV_WORK_COL_MMA(N) \ + } while (++j < jend); \ + GEMV_UNROLL(GEMV_DISASSEMBLE_MMA, N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) <= 1) { GEMV_UNROLL(GEMV_STORE_COL_MMA, N) } \ + else {GEMV_UNROLL_HALF(GEMV_LOAD2_COL_MMA, (N >> 1)) GEMV_UNROLL_HALF(GEMV_WORK2_COL_MMA, (N >> 1)) \ + GEMV_UNROLL_HALF(GEMV_STORE2_COL_MMA, (N >> 1))} i += (ResPacketSize * N); #endif -#define GEMV_INIT(iter, N) \ - if (N > iter) { \ - c##iter = pset1(ResScalar(0)); \ - } else { \ - EIGEN_UNUSED_VARIABLE(c##iter); \ +#define GEMV_INIT(iter, N) \ + EIGEN_IF_CONSTEXPR(N > iter) { c##iter = pset1(ResScalar(0)); } \ + else { \ + EIGEN_UNUSED_VARIABLE(c##iter); \ } #ifdef EIGEN_POWER_USE_GEMV_PREFETCH -#define GEMV_PREFETCH(iter, N) \ - if (GEMV_GETN(N) > ((iter >> 1) + ((N >> 1) * (iter & 1)))) { \ - lhs.prefetch(i + (iter * LhsPacketSize) + prefetch_dist, j); \ +#define GEMV_PREFETCH(iter, N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > ((iter >> 1) + ((N >> 1) * (iter & 1)))) { \ + lhs.prefetch(i + (iter * LhsPacketSize) + prefetch_dist, j); \ } #else #define GEMV_PREFETCH(iter, N) #endif -#define GEMV_WORK_COL(iter, N) \ - if (N > iter) { \ - c##iter = pcj.pmadd(GEMV_LOADPACKET_COL(iter), a0, c##iter); \ - } +#define GEMV_WORK_COL(iter, N) \ + EIGEN_IF_CONSTEXPR(N > iter) { c##iter = pcj.pmadd(GEMV_LOADPACKET_COL(iter), a0, c##iter); } #define GEMV_STORE_COL(iter, N) \ - if (N > iter) { \ + EIGEN_IF_CONSTEXPR(N > iter) { \ pstoreu(res + i + (iter * ResPacketSize), \ pmadd(c##iter, palpha, ploadu(res + i + (iter * ResPacketSize)))); \ } @@ -447,9 +437,8 @@ template EIGEN_ALWAYS_INLINE void outputVecCol(Packet4f acc, float* result, Packet4f pAlpha, Index extra_rows) { Packet4f d0 = ploadu(result); d0 = pmadd(acc, pAlpha, d0); - if (extraRows) { - pstoreu_partial(result, d0, extra_rows); - } else { + EIGEN_IF_CONSTEXPR(extraRows) { pstoreu_partial(result, d0, extra_rows); } + else { pstoreu(result, d0); } } @@ -461,9 +450,7 @@ EIGEN_ALWAYS_INLINE void outputVecColResults(Packet4f (&acc)[num_acc][size], flo for (Index k = 0; k < real_acc; k++) { outputVecCol(acc[k][0], result + k * 4, pAlpha, extra_rows); } - if (extraRows) { - outputVecCol(acc[real_acc][0], result + real_acc * 4, pAlpha, extra_rows); - } + EIGEN_IF_CONSTEXPR(extraRows) { outputVecCol(acc[real_acc][0], result + real_acc * 4, pAlpha, extra_rows); } } static Packet16uc p16uc_MERGE16_32_V1 = {0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17}; @@ -473,18 +460,16 @@ template EIGEN_ALWAYS_INLINE void loadVecLoopVSX(Index k, LhsMapper& lhs, Packet4f (&a0)[num_acc][2]) { Packet8bf c0 = lhs.template loadPacket(k * 4, 0); Packet8bf b1; - if (!zero) { + EIGEN_IF_CONSTEXPR(!zero) { b1 = lhs.template loadPacket(k * 4, 1); a0[k + 0][1] = oneConvertBF16Hi(b1.m_val); } a0[k + 0][0] = oneConvertBF16Hi(c0.m_val); - if (num_acc > (k + 1)) { + EIGEN_IF_CONSTEXPR(num_acc > (k + 1)) { a0[k + 1][0] = oneConvertBF16Lo(c0.m_val); - if (!zero) { - a0[k + 1][1] = oneConvertBF16Lo(b1.m_val); - } + EIGEN_IF_CONSTEXPR(!zero) { a0[k + 1][1] = oneConvertBF16Lo(b1.m_val); } } } @@ -530,9 +515,7 @@ EIGEN_ALWAYS_INLINE void vecColLoopVSX(Index j, LhsMapper& lhs, RhsMapper& rhs, Packet8bf b2 = loadColData(rhs, j); b0[0] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V1); - if (!zero) { - b0[1] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V2); - } + EIGEN_IF_CONSTEXPR(!zero) { b0[1] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V2); } using LhsSubMapper = typename LhsMapper::SubMapper; @@ -587,7 +570,7 @@ void colVSXVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, Rh template EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float* result) { - if (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { + EIGEN_IF_CONSTEXPR(MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { colVSXVecColLoopBody(row, cend, rows, lhs, rhs, pAlpha, result); } @@ -619,7 +602,7 @@ EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtra(Index& row, Index cend, Index colVSXVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); break; default: - if (extraRows) { + EIGEN_IF_CONSTEXPR(extraRows) { colVSXVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result); } break; @@ -645,15 +628,13 @@ EIGEN_ALWAYS_INLINE void calcVSXVecColLoops(Index cend, Index rows, LhsMapper& l template EIGEN_ALWAYS_INLINE void storeBF16fromResult(bfloat16* dst, Packet8bf data, Index resInc, Index extra) { if (inc) { - if (size < 8) { - pscatter_partial(dst + delta * resInc, data, resInc, extra); - } else { + EIGEN_IF_CONSTEXPR(size < 8) { pscatter_partial(dst + delta * resInc, data, resInc, extra); } + else { pscatter(dst + delta * resInc, data, resInc); } } else { - if (size < 8) { - pstoreu_partial(dst + delta, data, extra); - } else { + EIGEN_IF_CONSTEXPR(size < 8) { pstoreu_partial(dst + delta, data, extra); } + else { pstoreu(dst + delta, data); } } @@ -666,24 +647,20 @@ EIGEN_ALWAYS_INLINE void convertPointerF32toBF16VSX(Index& i, float* result, Ind while (i + size <= rows) { PacketBlock r32; r32.packet[0] = convertF32toBF16VSX(result + i + 0); - if (size >= 16) { - r32.packet[1] = convertF32toBF16VSX(result + i + 8); - } - if (size >= 32) { + EIGEN_IF_CONSTEXPR(size >= 16) { r32.packet[1] = convertF32toBF16VSX(result + i + 8); } + EIGEN_IF_CONSTEXPR(size >= 32) { r32.packet[2] = convertF32toBF16VSX(result + i + 16); r32.packet[3] = convertF32toBF16VSX(result + i + 24); } storeBF16fromResult(dst, r32.packet[0], resInc, rows & 7); - if (size >= 16) { - storeBF16fromResult(dst, r32.packet[1], resInc); - } - if (size >= 32) { + EIGEN_IF_CONSTEXPR(size >= 16) { storeBF16fromResult(dst, r32.packet[1], resInc); } + EIGEN_IF_CONSTEXPR(size >= 32) { storeBF16fromResult(dst, r32.packet[2], resInc); storeBF16fromResult(dst, r32.packet[3], resInc); } i += extra; dst += extra * resInc; - if (size != 32) break; + EIGEN_IF_CONSTEXPR(size != 32) break; } } @@ -766,9 +743,8 @@ EIGEN_ALWAYS_INLINE void outputVecResults(Packet4f (&acc)[num_acc][size], float* Packet4f d0 = ploadu(result + k); d0 = pmadd(acc[k + 0][0], pAlpha, d0); - if (num_acc > (k + 3)) { - pstoreu(result + k, d0); - } else { + EIGEN_IF_CONSTEXPR(num_acc > (k + 3)) { pstoreu(result + k, d0); } + else { if (extra == 3) { pstoreu_partial(result + k, d0, extra); } else { @@ -780,12 +756,13 @@ EIGEN_ALWAYS_INLINE void outputVecResults(Packet4f (&acc)[num_acc][size], float* template EIGEN_ALWAYS_INLINE void preduxVecResults2VSX(Packet4f (&acc)[num_acc][2], Index k) { - if (num_acc > (k + 1)) { + EIGEN_IF_CONSTEXPR(num_acc > (k + 1)) { acc[k][1] = vec_mergel(acc[k + 0][0], acc[k + 1][0]); acc[k][0] = vec_mergeh(acc[k + 0][0], acc[k + 1][0]); acc[k][0] = acc[k][0] + acc[k][1]; acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8); - } else { + } + else { acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8); #ifdef _BIG_ENDIAN acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12); @@ -799,7 +776,7 @@ template EIGEN_ALWAYS_INLINE void preduxVecResultsVSX(Packet4f (&acc)[num_acc][2]) { for (Index k = 0; k < num_acc; k += 4) { preduxVecResults2VSX(acc, k + 0); - if (num_acc > (k + 2)) { + EIGEN_IF_CONSTEXPR(num_acc > (k + 2)) { preduxVecResults2VSX(acc, k + 2); #ifdef EIGEN_VECTORIZE_VSX acc[k + 0][0] = reinterpret_cast( @@ -896,7 +873,7 @@ void colVSXVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMa template EIGEN_ALWAYS_INLINE void colVSXVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float* result) { - if (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { + EIGEN_IF_CONSTEXPR(MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { colVSXVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); } } @@ -1201,9 +1178,8 @@ EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine_row(std::complex* sr /** \internal load a scalar or a vector from complex location */ template EIGEN_ALWAYS_INLINE Packet4f pload_complex(std::complex* src) { - if (GEMV_IS_SCALAR) { - return pload_complex_half(src); - } else { + EIGEN_IF_CONSTEXPR(GEMV_IS_SCALAR) { return pload_complex_half(src); } + else { return ploadu(reinterpret_cast(src)); } } @@ -1264,9 +1240,8 @@ EIGEN_ALWAYS_INLINE Packet2d pload_real_full(std::complex* src) { /** \internal load a vector from a real-only scalar location - row-wise */ template EIGEN_ALWAYS_INLINE Packet4f pload_real_row(float* src) { - if (GEMV_IS_SCALAR) { - return pload_real_full(src); - } else { + EIGEN_IF_CONSTEXPR(GEMV_IS_SCALAR) { return pload_real_full(src); } + else { return ploadu(src); } } @@ -1330,10 +1305,11 @@ EIGEN_ALWAYS_INLINE Packet1cd pset_zero() { /** \internal initialize a vector from another vector */ template EIGEN_ALWAYS_INLINE Packet pset_init(Packet& c1) { - if (GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { EIGEN_UNUSED_VARIABLE(c1); return pset_zero(); - } else { + } + else { return c1; // Intentionally left uninitialized } } @@ -1361,11 +1337,12 @@ template EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, AlphaData& b0, ResScalar* res) { PResPacket c2 = pcplxflipconj(c0); - if (GEMV_IS_SCALAR) { + EIGEN_IF_CONSTEXPR(GEMV_IS_SCALAR) { ScalarPacket c4 = ploadu(reinterpret_cast(res)); ScalarPacket c3 = pmadd_complex(c0.v, c2.v, c4, b0); pstoreu(reinterpret_cast(res), c3); - } else { + } + else { ScalarPacket c4 = pload_complex(res); PResPacket c3 = PResPacket(pmadd_complex(c0.v, c2.v, c4, b0)); pstoreu(res, c3); @@ -1393,10 +1370,11 @@ EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, PResPacket& c1, A c6[1] = PResPacket(pmadd_complex(c1.v, c3.v, c6[1].v, b0)); GEMV_BUILDPAIR_MMA(a, c6[0].v, c6[1].v); #else - if (GEMV_IS_COMPLEX_FLOAT) { + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { __asm__("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.r.v), "wa"(c0.v), "wa"(c1.v)); __asm__("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.i.v), "wa"(c2.v), "wa"(c3.v)); - } else { + } + else { __asm__("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.r.v), "wa"(c0.v), "wa"(c1.v)); __asm__("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.i.v), "wa"(c2.v), "wa"(c3.v)); } @@ -1408,7 +1386,7 @@ EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, PResPacket& c1, A /** \internal load lhs packet */ template EIGEN_ALWAYS_INLINE LhsPacket loadLhsPacket(LhsMapper& lhs, Index i, Index j) { - if (sizeof(Scalar) == sizeof(LhsScalar)) { + EIGEN_IF_CONSTEXPR(sizeof(Scalar) == sizeof(LhsScalar)) { const LhsScalar& src = lhs(i + 0, j); return LhsPacket(pload_real_full(const_cast(&src))); } @@ -1418,11 +1396,11 @@ EIGEN_ALWAYS_INLINE LhsPacket loadLhsPacket(LhsMapper& lhs, Index i, Index j) { /** \internal madd for complex times complex */ template EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_complex(RealPacket& a, RealPacket& b, RealPacket& c) { - if (ConjugateLhs && ConjugateRhs) { - return vec_madd(a, pconj2(ComplexPacket(b)).v, c); - } else if (Negate && !ConjugateLhs && ConjugateRhs) { + EIGEN_IF_CONSTEXPR(ConjugateLhs && ConjugateRhs) { return vec_madd(a, pconj2(ComplexPacket(b)).v, c); } + else EIGEN_IF_CONSTEXPR(Negate && !ConjugateLhs && ConjugateRhs) { return vec_nmsub(a, b, c); - } else { + } + else { return vec_madd(a, b, c); } } @@ -1430,9 +1408,8 @@ EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_complex(RealPacket& a, RealPacket& /** \internal madd for complex times real */ template EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_real(RealPacket& a, RealPacket& b, RealPacket& c) { - if (Conjugate) { - return vec_madd(a, pconj2(ComplexPacket(b)).v, c); - } else { + EIGEN_IF_CONSTEXPR(Conjugate) { return vec_madd(a, pconj2(ComplexPacket(b)).v, c); } + else { return vec_madd(a, b, c); } } @@ -1442,9 +1419,8 @@ template pcj; RhsPacket b0; - if (StorageOrder == ColMajor) { - b0 = pset1(*b); - } else { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pset1(*b); } + else { b0 = ploadu(b); } c0 = pcj.pmadd(a0, b0, c0); @@ -1455,12 +1431,11 @@ template EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0, ResPacket& c1) { ScalarPacket br, bi; - if (StorageOrder == ColMajor) { - pload_realimag(b, br, bi); - } else { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { pload_realimag(b, br, bi); } + else { pload_realimag_row(b, br, bi); } - if (ConjugateLhs && !ConjugateRhs) a0 = pconj2(a0); + EIGEN_IF_CONSTEXPR(ConjugateLhs && !ConjugateRhs) a0 = pconj2(a0); LhsPacket a1 = pcplxflipconj(a0); ScalarPacket cr = pmadd_complex_complex(a0.v, br, c0.v); ScalarPacket ci = pmadd_complex_complex(a1.v, bi, c1.v); @@ -1473,9 +1448,8 @@ template EIGEN_ALWAYS_INLINE void gemv_mult_real_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0) { ScalarPacket b0; - if (StorageOrder == ColMajor) { - b0 = pload_complex_full(b); - } else { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_complex_full(b); } + else { b0 = pload_complex_full_row(b); } ScalarPacket cri = pmadd_complex_real(a0, b0, c0.v); @@ -1488,9 +1462,8 @@ template (&a0); ScalarPacket b0; - if (StorageOrder == ColMajor) { - b0 = pload_real(b); - } else { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_real(b); } + else { b0 = pload_real_row(b); } ScalarPacket cri = pmadd_complex_real(a1, b0, c0.v); @@ -1569,9 +1542,10 @@ EIGEN_ALWAYS_INLINE void pload_complex_MMA(__vector_pair&) { /** \internal perform a matrix multiply and accumulate (positive and negative) of packet a and packet b */ template EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, RhsPacket& a, LhsPacket& b) { - if (NegativeAccumulate) { + EIGEN_IF_CONSTEXPR(NegativeAccumulate) { __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b); - } else { + } + else { __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b); } } @@ -1579,9 +1553,8 @@ EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, RhsPacket& a, LhsPacket /** \internal perform a matrix multiply and accumulate (positive and negative) of vector_pair a and packet b */ template EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, __vector_pair& a, Packet2d& b) { - if (NegativeAccumulate) { - __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b); - } else { + EIGEN_IF_CONSTEXPR(NegativeAccumulate) { __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b); } + else { __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b); } } @@ -1594,24 +1567,28 @@ EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad*, __vector_pair&, Packet4f&) /** \internal madd for complex times complex (MMA version) */ template EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(LhsPacket& a, RealPacket& b, __vector_quad* c) { - if (ConjugateLhs && ConjugateRhs) { + EIGEN_IF_CONSTEXPR(ConjugateLhs && ConjugateRhs) { RealPacket b2 = pconj2(convertComplex(b)).v; return pger_vecMMA(c, b2, a.v); - } else if (Negate && !ConjugateLhs && ConjugateRhs) { + } + else EIGEN_IF_CONSTEXPR(Negate && !ConjugateLhs && ConjugateRhs) { return pger_vecMMA(c, b, a.v); - } else { + } + else { return pger_vecMMA(c, b, a.v); } } template EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(__vector_pair& a, RealPacket& b, __vector_quad* c) { - if (ConjugateLhs && ConjugateRhs) { + EIGEN_IF_CONSTEXPR(ConjugateLhs && ConjugateRhs) { RealPacket b2 = pconj2(convertComplex(b)).v; return pger_vecMMA(c, a, b2); - } else if (Negate && !ConjugateLhs && ConjugateRhs) { + } + else EIGEN_IF_CONSTEXPR(Negate && !ConjugateLhs && ConjugateRhs) { return pger_vecMMA(c, a, b); - } else { + } + else { return pger_vecMMA(c, a, b); } } @@ -1620,17 +1597,16 @@ EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(__vector_pair& a, RealPacket& template EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(LhsPacket& a, RealPacket& b, __vector_quad* c) { RealPacket a2 = convertReal(a); - if (Conjugate) { + EIGEN_IF_CONSTEXPR(Conjugate) { RealPacket b2 = pconj2(convertComplex(b)).v; - if (StorageOrder == ColMajor) { - return pger_vecMMA(c, b2, a2); - } else { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { return pger_vecMMA(c, b2, a2); } + else { return pger_vecMMA(c, a2, b2); } - } else { - if (StorageOrder == ColMajor) { - return pger_vecMMA(c, b, a2); - } else { + } + else { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { return pger_vecMMA(c, b, a2); } + else { return pger_vecMMA(c, a2, b); } } @@ -1639,10 +1615,11 @@ EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(LhsPacket& a, RealPacket& b, __v /** \internal madd for real times complex (MMA version) */ template EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(__vector_pair& a, RealPacket& b, __vector_quad* c) { - if (Conjugate) { + EIGEN_IF_CONSTEXPR(Conjugate) { RealPacket b2 = pconj2(convertComplex(b)).v; return pger_vecMMA(c, a, b2); - } else { + } + else { return pger_vecMMA(c, a, b); } } @@ -1652,9 +1629,8 @@ template EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) { ScalarPacket b0; - if (StorageOrder == ColMajor) { - b0 = pload_realimag_combine(b); - } else { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_realimag_combine(b); } + else { b0 = pload_realimag_combine_row(b); } pmadd_complex_complex_MMA(a0, b0, c0); @@ -1666,9 +1642,8 @@ template (a0); ScalarPacket b0; - if (StorageOrder == ColMajor) { - b0 = pload_real(b); - } else { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_real(b); } + else { b0 = pload_real_row(b); } pmadd_complex_real_MMA(a0, b0, c0); @@ -1679,9 +1654,8 @@ template EIGEN_ALWAYS_INLINE void gemv_mult_real_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) { ScalarPacket b0; - if (StorageOrder == ColMajor) { - b0 = pload_complex_full(b); - } else { + EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_complex_full(b); } + else { b0 = pload_complex_full_row(b); } pmadd_complex_real_MMA) template EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(__vector_pair& a0, std::complex* b, __vector_quad* c0) { - if (sizeof(LhsScalar) == 16) { + EIGEN_IF_CONSTEXPR(sizeof(LhsScalar) == 16) { gemv_mult_complex_complex_MMA(a0, b, c0); - } else { + } + else { gemv_mult_real_complex_MMA(a0, b, c0); } @@ -1742,8 +1717,8 @@ template EIGEN_ALWAYS_INLINE void disassembleResults2(__vector_quad* c0, PacketBlock& result0) { __builtin_mma_disassemble_acc(&result0.packet, c0); - if (sizeof(LhsPacket) == 16) { - if (sizeof(RhsPacket) == 16) { + EIGEN_IF_CONSTEXPR(sizeof(LhsPacket) == 16) { + EIGEN_IF_CONSTEXPR(sizeof(RhsPacket) == 16) { ScalarPacket tmp0, tmp2; tmp2 = vec_mergeh(result0.packet[2], result0.packet[3]); tmp0 = vec_mergeh(result0.packet[0], result0.packet[1]); @@ -1752,19 +1727,22 @@ EIGEN_ALWAYS_INLINE void disassembleResults2(__vector_quad* c0, PacketBlock EIGEN_ALWAYS_INLINE void disassembleResults4(__vector_quad* c0, PacketBlock& result0) { __builtin_mma_disassemble_acc(&result0.packet, c0); - if (GEMV_IS_COMPLEX_COMPLEX) { - if (ConjugateLhs) { + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR(ConjugateLhs) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v; - } else { - if (ConjugateRhs) { - result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; - } else { + } + else { + EIGEN_IF_CONSTEXPR(ConjugateRhs) { result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; } + else { result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v; } } result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); - } else if (sizeof(LhsPacket) == sizeof(std::complex)) { - if (ConjugateLhs) { - result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; - } - } else { + } + else EIGEN_IF_CONSTEXPR(sizeof(LhsPacket) == sizeof(std::complex)) { + EIGEN_IF_CONSTEXPR(ConjugateLhs) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; } + } + else { result0.packet[0] = vec_mergee(result0.packet[0], result0.packet[1]); } } @@ -1799,9 +1777,10 @@ EIGEN_ALWAYS_INLINE void disassembleResults4(__vector_quad* c0, PacketBlock EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock& result0) { - if (!GEMV_IS_COMPLEX_FLOAT) { + EIGEN_IF_CONSTEXPR(!GEMV_IS_COMPLEX_FLOAT) { disassembleResults2(c0, result0); - } else { + } + else { disassembleResults4(c0, result0); } } @@ -1816,9 +1795,7 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter) { \ - __builtin_mma_xxsetaccz(&e0##iter); \ - } + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { __builtin_mma_xxsetaccz(&e0##iter); } #if EIGEN_COMP_LLVM #define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \ @@ -1827,35 +1804,39 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock(const_cast(&src)); \ EIGEN_UNUSED_VARIABLE(f##iter1); \ - } else { \ + } \ + else { \ f##iter1 = lhs.template load(i + ((iter2) * ResPacketSize), j); \ GEMV_BUILDPAIR_MMA(a##iter1, vec_splat(convertReal(f##iter1), 0), vec_splat(convertReal(f##iter1), 1)); \ } #endif #define GEMV_LOAD1_COL_COMPLEX_MMA(iter, N) \ - if (GEMV_GETN_COMPLEX(N) > iter) { \ - if (GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \ EIGEN_UNUSED_VARIABLE(a##iter); \ - } else { \ + } \ + else { \ GEMV_LOADPAIR_COL_COMPLEX_MMA(iter, iter << 1) \ } \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(a##iter); \ EIGEN_UNUSED_VARIABLE(f##iter); \ } #define GEMV_WORK1_COL_COMPLEX_MMA(iter, N) \ - if (GEMV_GETN_COMPLEX(N) > iter) { \ - if (GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ gemv_mult_complex_MMA(f##iter, b, &e0##iter); \ - } else { \ + } \ + else { \ gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ } \ @@ -1865,15 +1846,17 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter1) { \ - if (GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2); \ EIGEN_UNUSED_VARIABLE(a##iter3); \ - } else { \ + } \ + else { \ GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2 << 1); \ GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter3, iter3 << 1); \ } \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(a##iter2); \ EIGEN_UNUSED_VARIABLE(a##iter3); \ } \ @@ -1881,15 +1864,16 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter1) { \ - if (GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ PLhsPacket g[2]; \ __builtin_vsx_disassemble_pair(reinterpret_cast(g), &a##iter2); \ gemv_mult_complex_MMA(g[0], b, &e0##iter2); \ gemv_mult_complex_MMA(g[1], b, &e0##iter3); \ - } else { \ + } \ + else { \ gemv_mult_complex_MMA(a##iter2, b, &e0##iter2); \ gemv_mult_complex_MMA 1) { \ - GEMV_UNROLL_HALF(GEMV_LOAD2_COL_COMPLEX_MMA, (N >> 1)) \ - } else { \ - GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) \ +#define GEMV_LOAD_COL_COMPLEX_MMA(N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > 1) { GEMV_UNROLL_HALF(GEMV_LOAD2_COL_COMPLEX_MMA, (N >> 1)) } \ + else { \ + GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) \ } -#define GEMV_WORK_COL_COMPLEX_MMA(N) \ - if (GEMV_GETN_COMPLEX(N) > 1) { \ - GEMV_UNROLL_HALF(GEMV_WORK2_COL_COMPLEX_MMA, (N >> 1)) \ - } else { \ - GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) \ +#define GEMV_WORK_COL_COMPLEX_MMA(N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > 1) { GEMV_UNROLL_HALF(GEMV_WORK2_COL_COMPLEX_MMA, (N >> 1)) } \ + else { \ + GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) \ } #else #define GEMV_LOAD_COL_COMPLEX_MMA(N) GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) @@ -1922,13 +1904,14 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { \ GEMV_DISASSEMBLE_COMPLEX_MMA(iter); \ c0##iter = PResPacket(result0##iter.packet[0]); \ - if (GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ pstoreu_pmadd_complex( \ c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \ - } else { \ + } \ + else { \ pstoreu_pmadd_complex( \ c0##iter, alpha_data, res + i + ((iter << 1) * ResPacketSize)); \ c0##iter = PResPacket(result0##iter.packet[2]); \ @@ -1938,15 +1921,16 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter1) { \ GEMV_DISASSEMBLE_COMPLEX_MMA(iter2); \ GEMV_DISASSEMBLE_COMPLEX_MMA(iter3); \ c0##iter2 = PResPacket(result0##iter2.packet[0]); \ - if (GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ c0##iter3 = PResPacket(result0##iter3.packet[0]); \ pstoreu_pmadd_complex( \ c0##iter2, c0##iter3, alpha_data, res + i); \ - } else { \ + } \ + else { \ c0##iter3 = PResPacket(result0##iter2.packet[2]); \ pstoreu_pmadd_complex( \ c0##iter2, c0##iter3, alpha_data, res + i); \ @@ -1957,47 +1941,43 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock(&b1); \ - GEMV_UNROLL(GEMV_PREFETCH, N) \ - GEMV_LOAD_COL_COMPLEX_MMA(N) \ - GEMV_WORK_COL_COMPLEX_MMA(N) \ - } while (++j < jend); \ - if (GEMV_GETN(N) <= 2) { \ - GEMV_UNROLL(GEMV_STORE_COL_COMPLEX_MMA, N) \ - } else { \ - GEMV_UNROLL_HALF(GEMV_STORE2_COL_COMPLEX_MMA, (N >> 1)) \ - } \ - i += (ResPacketSize * N); +#define GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \ + GEMV_UNROLL(GEMV_INIT_COL_COMPLEX_MMA, N) \ + Index j = j2; \ + do { \ + const RhsScalar& b1 = rhs2(j, 0); \ + RhsScalar* b = const_cast(&b1); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_LOAD_COL_COMPLEX_MMA(N) \ + GEMV_WORK_COL_COMPLEX_MMA(N) \ + } while (++j < jend); \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) <= 2) { GEMV_UNROLL(GEMV_STORE_COL_COMPLEX_MMA, N) } \ + else {GEMV_UNROLL_HALF(GEMV_STORE2_COL_COMPLEX_MMA, (N >> 1))} i += (ResPacketSize * N); #endif #define GEMV_INIT_COMPLEX(iter, N) \ - if (N > iter) { \ + EIGEN_IF_CONSTEXPR(N > iter) { \ c0##iter = pset_zero(); \ c1##iter = pset_init(c1##iter); \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(c0##iter); \ EIGEN_UNUSED_VARIABLE(c1##iter); \ } #define GEMV_WORK_COL_COMPLEX(iter, N) \ - if (N > iter) { \ + EIGEN_IF_CONSTEXPR(N > iter) { \ f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \ gemv_mult_complex(f##iter, b, c0##iter, c1##iter); \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(f##iter); \ } #define GEMV_STORE_COL_COMPLEX(iter, N) \ - if (N > iter) { \ - if (GEMV_IS_COMPLEX_COMPLEX) { \ - c0##iter = padd(c0##iter, c1##iter); \ - } \ + EIGEN_IF_CONSTEXPR(N > iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { c0##iter = padd(c0##iter, c1##iter); } \ pstoreu_pmadd_complex( \ c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \ } @@ -2023,11 +2003,10 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock 10) -#define GEMV_PROCESS_COL_COMPLEX(N) \ - if (sizeof(Scalar) != sizeof(LhsPacket)) { \ - GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \ - } else { \ - GEMV_PROCESS_COL_COMPLEX_ONE(N) \ +#define GEMV_PROCESS_COL_COMPLEX(N) \ + EIGEN_IF_CONSTEXPR(sizeof(Scalar) != sizeof(LhsPacket)) { GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) } \ + else { \ + GEMV_PROCESS_COL_COMPLEX_ONE(N) \ } #else #define GEMV_PROCESS_COL_COMPLEX(N) GEMV_PROCESS_COL_COMPLEX_ONE(N) @@ -2100,7 +2079,7 @@ EIGEN_STRONG_INLINE void gemv_complex_col(Index rows, Index cols, const LhsMappe GEMV_UNUSED(8, a) GEMV_UNUSED(8, f) #if !defined(GCC_ONE_VECTORPAIR_BUG) && defined(USE_GEMV_COL_COMPLEX_MMA) - if (GEMV_IS_COMPLEX_COMPLEX || !GEMV_IS_COMPLEX_FLOAT) + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX || !GEMV_IS_COMPLEX_FLOAT) #endif #endif #ifndef GCC_ONE_VECTORPAIR_BUG @@ -2177,23 +2156,26 @@ EIGEN_ALWAYS_INLINE ScalarBlock, 2> addComplexResults(Packet result0.packet[2] = reinterpret_cast( vec_mergel(reinterpret_cast(result0.packet[2]), reinterpret_cast(result1.packet[2]))); result0.packet[0] = vec_add(result0.packet[0], result0.packet[2]); - if (GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { result0.packet[1] = reinterpret_cast( vec_mergeh(reinterpret_cast(result0.packet[1]), reinterpret_cast(result1.packet[1]))); result0.packet[3] = reinterpret_cast( vec_mergel(reinterpret_cast(result0.packet[3]), reinterpret_cast(result1.packet[3]))); result0.packet[1] = vec_add(result0.packet[1], result0.packet[3]); - if (ConjugateLhs) { + EIGEN_IF_CONSTEXPR(ConjugateLhs) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v; - } else if (ConjugateRhs) { + } + else EIGEN_IF_CONSTEXPR(ConjugateRhs) { result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; - } else { + } + else { result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v; } result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); - } else { - if (ConjugateLhs && (sizeof(LhsPacket) == sizeof(std::complex))) { + } + else { + EIGEN_IF_CONSTEXPR(ConjugateLhs && (sizeof(LhsPacket) == sizeof(std::complex))) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; } } @@ -2237,20 +2219,23 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(__vector_quad* acc0 ScalarBlock cc0; PacketBlock result0; __builtin_mma_disassemble_acc(&result0.packet, acc0); - if (GEMV_IS_COMPLEX_COMPLEX) { - if (ConjugateLhs) { + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR(ConjugateLhs) { result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v; result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v; - } else if (ConjugateRhs) { + } + else EIGEN_IF_CONSTEXPR(ConjugateRhs) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v; - } else { + } + else { result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v; result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v; } result0.packet[0] = vec_add(result0.packet[0], __builtin_vsx_xxpermdi(result0.packet[1], result0.packet[1], 2)); result0.packet[2] = vec_add(result0.packet[2], __builtin_vsx_xxpermdi(result0.packet[3], result0.packet[3], 2)); - } else { + } + else { result0.packet[0] = __builtin_vsx_xxpermdi(result0.packet[0], result0.packet[1], 1); result0.packet[2] = __builtin_vsx_xxpermdi(result0.packet[2], result0.packet[3], 1); } @@ -2288,64 +2273,59 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(ResPacket& a, ResPa #define GEMV_UNUSED_ROW(N, which) GEMV_UNROLL3_ROW(GEMV_UNUSED_VAR, N, which) -#define GEMV_INIT_ROW(iter, N) \ - if (GEMV_GETN(N) > iter) { \ - __builtin_mma_xxsetaccz(&c##iter); \ - } +#define GEMV_INIT_ROW(iter, N) \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { __builtin_mma_xxsetaccz(&c##iter); } #define GEMV_LOADPAIR_ROW(iter1, iter2) \ GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_ROW(iter2), GEMV_LOADPACKET_ROW((iter2) + 1)); #define GEMV_WORK_ROW(iter, N) \ - if (GEMV_GETN(N) > iter) { \ - if (GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ pger_vecMMA_acc(&c##iter, a0, GEMV_LOADPACKET_ROW(iter)); \ - } else { \ + } \ + else { \ __vector_pair b##iter; \ GEMV_LOADPAIR_ROW(iter, iter << 1) \ pger_vecMMA_acc(&c##iter, b##iter, a0); \ } \ } -#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ - if (N > iter1) { \ - if (GEMV_IS_FLOAT) { \ - cc##iter1 = predux_real(&c##iter2, &c##iter3); \ - } else { \ - cc##iter1 = predux_real(&c##iter1); \ - } \ - } else { \ - EIGEN_UNUSED_VARIABLE(cc##iter1); \ +#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ + EIGEN_IF_CONSTEXPR(N > iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { cc##iter1 = predux_real(&c##iter2, &c##iter3); } \ + else { \ + cc##iter1 = predux_real(&c##iter1); \ + } \ + } \ + else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ } #else -#define GEMV_INIT_ROW(iter, N) \ - if (N > iter) { \ - c##iter = pset1(ResScalar(0)); \ - } else { \ - EIGEN_UNUSED_VARIABLE(c##iter); \ +#define GEMV_INIT_ROW(iter, N) \ + EIGEN_IF_CONSTEXPR(N > iter) { c##iter = pset1(ResScalar(0)); } \ + else { \ + EIGEN_UNUSED_VARIABLE(c##iter); \ } -#define GEMV_WORK_ROW(iter, N) \ - if (N > iter) { \ - c##iter = pcj.pmadd(GEMV_LOADPACKET_ROW(iter), a0, c##iter); \ - } +#define GEMV_WORK_ROW(iter, N) \ + EIGEN_IF_CONSTEXPR(N > iter) { c##iter = pcj.pmadd(GEMV_LOADPACKET_ROW(iter), a0, c##iter); } -#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ - if (N > iter1) { \ - cc##iter1 = predux_real(c##iter2, c##iter3); \ - } else { \ - EIGEN_UNUSED_VARIABLE(cc##iter1); \ +#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ + EIGEN_IF_CONSTEXPR(N > iter1) { cc##iter1 = predux_real(c##iter2, c##iter3); } \ + else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ } #endif #define GEMV_MULT(iter1, iter2, iter3, N) \ - if (N > iter1) { \ + EIGEN_IF_CONSTEXPR(N > iter1) { \ cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), a0); \ cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), a0); \ } #define GEMV_STORE_ROW(iter1, iter2, iter3, N) \ - if (N > iter1) { \ + EIGEN_IF_CONSTEXPR(N > iter1) { \ storeMaddData(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \ storeMaddData(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \ } @@ -2502,7 +2482,7 @@ EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16() template EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PResPacket& b0, ResPacket& a1, ResPacket& b1) { - if (GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { a0 = padd(a0, a1); b0 = padd(b0, b1); } @@ -2530,20 +2510,19 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PRe #ifdef USE_GEMV_MMA #define GEMV_INIT_ROW_COMPLEX_MMA(iter, N) \ - if (GEMV_GETN_COMPLEX(N) > iter) { \ - __builtin_mma_xxsetaccz(&e0##iter); \ - } + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { __builtin_mma_xxsetaccz(&e0##iter); } #define GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter1, iter2) \ GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter2), GEMV_LOADPACKET_ROW_COMPLEX_DATA((iter2) + 1)); #define GEMV_WORK_ROW_COMPLEX_MMA(iter, N) \ - if (GEMV_GETN_COMPLEX(N) > iter) { \ - if (GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \ gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ - } else { \ + } \ + else { \ __vector_pair a##iter; \ GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter, iter << 1) \ gemv_mult_complex_MMA predux_complex(PResPacket& a0, PRe } #define GEMV_PREDUX4_COMPLEX_MMA(iter1, iter2, iter3, N) \ - if (N > iter1) { \ - if (GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR(N > iter1) { \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ cc##iter1 = predux_complex( \ &e0##iter2, &e0##iter3); \ - } else { \ + } \ + else { \ cc##iter1 = \ predux_complex(&e0##iter1); \ } \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(cc##iter1); \ } @@ -2577,28 +2558,29 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PRe #endif #define GEMV_WORK_ROW_COMPLEX(iter, N) \ - if (N > iter) { \ + EIGEN_IF_CONSTEXPR(N > iter) { \ PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \ gemv_mult_complex(a##iter, b, c0##iter, c1##iter); \ } #define GEMV_PREDUX4_COMPLEX(iter1, iter2, iter3, N) \ - if (N > iter1) { \ + EIGEN_IF_CONSTEXPR(N > iter1) { \ cc##iter1 = predux_complex(c0##iter2, c0##iter3, \ c1##iter2, c1##iter3); \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(cc##iter1); \ } #define GEMV_MULT_COMPLEX(iter1, iter2, iter3, N) \ - if (N > iter1) { \ + EIGEN_IF_CONSTEXPR(N > iter1) { \ cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), b0); \ cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), b0); \ } #define GEMV_STORE_ROW_COMPLEX(iter1, iter2, iter3, N) \ - if (N > iter1) { \ + EIGEN_IF_CONSTEXPR(N > iter1) { \ storeMaddData(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \ storeMaddData(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \ } @@ -2616,10 +2598,8 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PRe GEMV_PROCESS_END_ROW_COMPLEX(N); \ } -#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \ - if (GEMV_IS_COMPLEX_COMPLEX) { \ - c0##iter = padd(c0##iter, c1##iter); \ - } \ +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \ + EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { c0##iter = padd(c0##iter, c1##iter); } \ dd0 = predux(c0##iter); #if EIGEN_COMP_LLVM @@ -2634,25 +2614,25 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PRe #define GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter) lhs.template load(i + (iter), j) -#define GEMV_INIT_COMPLEX_OLD(iter, N) \ - EIGEN_UNUSED_VARIABLE(c0##iter); \ - if (N > iter) { \ - c1##iter = pset_zero(); \ - } else { \ - EIGEN_UNUSED_VARIABLE(c1##iter); \ +#define GEMV_INIT_COMPLEX_OLD(iter, N) \ + EIGEN_UNUSED_VARIABLE(c0##iter); \ + EIGEN_IF_CONSTEXPR(N > iter) { c1##iter = pset_zero(); } \ + else { \ + EIGEN_UNUSED_VARIABLE(c1##iter); \ } #define GEMV_WORK_ROW_COMPLEX_OLD(iter, N) \ - if (N > iter) { \ + EIGEN_IF_CONSTEXPR(N > iter) { \ LhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter); \ c1##iter = pcj.pmadd(a##iter, b0, c1##iter); \ } #define GEMV_PREDUX4_COMPLEX_OLD(iter1, iter2, iter3, N) \ - if (N > iter1) { \ + EIGEN_IF_CONSTEXPR(N > iter1) { \ cc##iter1.scalar[0] = predux(c1##iter2); \ cc##iter1.scalar[1] = predux(c1##iter3); \ - } else { \ + } \ + else { \ EIGEN_UNUSED_VARIABLE(cc##iter1); \ } @@ -2763,15 +2743,11 @@ EIGEN_STRONG_INLINE void gemv_complex_row(Index rows, Index cols, const LhsMappe #ifndef GCC_ONE_VECTORPAIR_BUG ScalarBlock cc0, cc1, cc2, cc3; #ifdef USE_GEMV_MMA - if (!GEMV_IS_COMPLEX_COMPLEX) + EIGEN_IF_CONSTEXPR(!GEMV_IS_COMPLEX_COMPLEX) #endif - { - GEMV_PROCESS_ROW_COMPLEX(8) - } - GEMV_PROCESS_ROW_COMPLEX(4) - GEMV_PROCESS_ROW_COMPLEX(2) + {GEMV_PROCESS_ROW_COMPLEX(8)} GEMV_PROCESS_ROW_COMPLEX(4) GEMV_PROCESS_ROW_COMPLEX(2) #endif - for (; i < rows; ++i) { + for (; i < rows; ++i) { GEMV_PROCESS_ROW_COMPLEX_SINGLE(1) GEMV_PROCESS_ROW_COMPLEX_PREDUX(0) for (; j < cols; ++j) { diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h index 435d933e5..48567d63a 100644 --- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h @@ -1846,9 +1846,9 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= 3 * PacketSize) { + EIGEN_IF_CONSTEXPR(Pack1 >= 3 * PacketSize) { for (; i < peeled_mc3; i += 3 * PacketSize) { - if (PanelMode) count += (3 * PacketSize) * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += (3 * PacketSize) * offset; for (Index k = 0; k < depth; k++) { Packet A, B, C; @@ -1862,13 +1862,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= 2 * PacketSize) { + EIGEN_IF_CONSTEXPR(Pack1 >= 2 * PacketSize) { for (; i < peeled_mc2; i += 2 * PacketSize) { - if (PanelMode) count += (2 * PacketSize) * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += (2 * PacketSize) * offset; for (Index k = 0; k < depth; k++) { Packet A, B; @@ -1879,13 +1879,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= 1 * PacketSize) { + EIGEN_IF_CONSTEXPR(Pack1 >= 1 * PacketSize) { for (; i < peeled_mc1; i += 1 * PacketSize) { - if (PanelMode) count += (1 * PacketSize) * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += (1 * PacketSize) * offset; for (Index k = 0; k < depth; k++) { Packet A; @@ -1893,13 +1893,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= HalfPacketSize) { + EIGEN_IF_CONSTEXPR(HasHalf && Pack1 >= HalfPacketSize) { for (; i < peeled_mc_half; i += HalfPacketSize) { - if (PanelMode) count += (HalfPacketSize)*offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += (HalfPacketSize)*offset; for (Index k = 0; k < depth; k++) { HalfPacket A; @@ -1907,13 +1907,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= QuarterPacketSize) { + EIGEN_IF_CONSTEXPR(HasQuarter && Pack1 >= QuarterPacketSize) { for (; i < peeled_mc_quarter; i += QuarterPacketSize) { - if (PanelMode) count += (QuarterPacketSize)*offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += (QuarterPacketSize)*offset; for (Index k = 0; k < depth; k++) { QuarterPacket A; @@ -1921,7 +1921,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs 1) { + EIGEN_IF_CONSTEXPR(Pack2 < PacketSize && Pack2 > 1) { const Index pack2_progress = (HasHalf || HasQuarter) ? last_lhs_progress : Pack2; const Index peeled = (HasHalf || HasQuarter) ? peeled_mc0 : (rows / Pack2) * Pack2; for (; i < peeled; i += pack2_progress) { - if (PanelMode) count += pack2_progress * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += pack2_progress * offset; for (Index k = 0; k < depth; k++) for (Index w = 0; w < pack2_progress; w++) blockA[count++] = cj(lhs(i + w, k)); - if (PanelMode) count += pack2_progress * (stride - offset - depth); + EIGEN_IF_CONSTEXPR(PanelMode) count += pack2_progress * (stride - offset - depth); } } // Pack scalars for (; i < rows; i++) { - if (PanelMode) count += offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += offset; for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k)); - if (PanelMode) count += (stride - offset - depth); + EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth); } } @@ -1996,7 +1996,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs 1 ? (rows / pack) * pack : 0 : i + (remaining_rows / pack) * pack; Index starting_pos = i; for (; i < peeled_mc; i += pack) { - if (PanelMode) count += pack * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += pack * offset; Index k = 0; if (pack >= psize && psize >= QuarterPacketSize) { @@ -2042,7 +2042,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { // skip what we have before - if (PanelMode) count += 8 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset; const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); @@ -2124,10 +2126,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs kernel0, kernel1, kernel2, kernel3; kernel0.packet[0 % PacketSize] = dm0.template loadPacket(k); kernel0.packet[1 % PacketSize] = dm1.template loadPacket(k); @@ -2152,7 +2154,8 @@ EIGEN_DONT_INLINE void gemm_pack_rhs kernel0, kernel1; kernel0.packet[0 % PacketSize] = dm0.template loadPacket(k); @@ -2175,7 +2178,8 @@ EIGEN_DONT_INLINE void gemm_pack_rhs kernel0; kernel0.packet[0 % PacketSize] = dm0.template loadPacket(k); @@ -2213,14 +2217,14 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= 4) { for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { // skip what we have before - if (PanelMode) count += 4 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset; const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); @@ -2269,19 +2273,19 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= depth && offset <= stride)); - const bool HasHalf = (int)HalfPacketSize < (int)PacketSize; - const bool HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize; + constexpr bool HasHalf = (int)HalfPacketSize < (int)PacketSize; + constexpr bool HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize; conj_if::IsComplex && Conjugate> cj; Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0; Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0; @@ -2313,19 +2317,21 @@ struct gemm_pack_rhs= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { // skip what we have before - if (PanelMode) count += 8 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset; for (Index k = 0; k < depth; k++) { - if (PacketSize == 8) { + EIGEN_IF_CONSTEXPR(PacketSize == 8) { Packet A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += PacketSize; - } else if (PacketSize == 4) { + } + else EIGEN_IF_CONSTEXPR(PacketSize == 4) { Packet A = rhs.template loadPacket(k, j2); Packet B = rhs.template loadPacket(k, j2 + 4); pstoreu(blockB + count, cj.pconj(A)); pstoreu(blockB + count + PacketSize, cj.pconj(B)); count += 2 * PacketSize; - } else { + } + else { const LinearMapper dm0 = rhs.getLinearMapper(k, j2); blockB[count + 0] = cj(dm0(0)); blockB[count + 1] = cj(dm0(1)); @@ -2339,28 +2345,31 @@ struct gemm_pack_rhs= 4) { + EIGEN_IF_CONSTEXPR(nr >= 4) { for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { // skip what we have before - if (PanelMode) count += 4 * offset; + EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset; for (Index k = 0; k < depth; k++) { - if (PacketSize == 4) { + EIGEN_IF_CONSTEXPR(PacketSize == 4) { Packet A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += PacketSize; - } else if (HasHalf && HalfPacketSize == 4) { + } + else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 4) { HalfPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += HalfPacketSize; - } else if (HasQuarter && QuarterPacketSize == 4) { + } + else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 4) { QuarterPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += QuarterPacketSize; - } else { + } + else { const LinearMapper dm0 = rhs.getLinearMapper(k, j2); blockB[count + 0] = cj(dm0(0)); blockB[count + 1] = cj(dm0(1)); @@ -2370,17 +2379,17 @@ struct gemm_pack_rhs processed with gebp or skipped // 2 - the actual_mc x actual_mc symmetric block => processed with a special kernel // 3 - after the diagonal => processed with gebp or skipped - if (UpLo == Lower) + EIGEN_IF_CONSTEXPR(UpLo == Lower) { gebp(res.getSubMapper(i2, 0), blockA, blockB, actual_mc, actual_kc, (std::min)(size, i2), alpha, -1, -1, 0, 0); + } sybb(res_ + resStride * i2 + resIncr * i2, resIncr, resStride, blockA, blockB + actual_kc * i2, actual_mc, actual_kc, alpha); - if (UpLo == Upper) { + EIGEN_IF_CONSTEXPR(UpLo == Upper) { Index j2 = i2 + actual_mc; gebp(res.getSubMapper(i2, j2), blockA, blockB + actual_kc * j2, actual_mc, actual_kc, (std::max)(Index(0), size - j2), alpha, -1, -1, 0, 0); @@ -170,8 +171,9 @@ struct tribb_kernel { Index actualBlockSize = std::min(BlockSize, size - j); const RhsScalar* actual_b = blockB + j * depth; - if (UpLo == Upper) + EIGEN_IF_CONSTEXPR(UpLo == Upper) { gebp_kernel1(res.getSubMapper(0, j), blockA, actual_b, j, depth, actualBlockSize, alpha, -1, -1, 0, 0); + } // selfadjoint micro block { @@ -189,7 +191,7 @@ struct tribb_kernel { } } - if (UpLo == Lower) { + EIGEN_IF_CONSTEXPR(UpLo == Lower) { Index i = j + actualBlockSize; gebp_kernel1(res.getSubMapper(i, j), blockA + depth * i, actual_b, size - i, depth, actualBlockSize, alpha, -1, -1, 0, 0); @@ -238,14 +240,18 @@ struct general_product_to_triangular_selector(actualLhs.data()) : static_lhs.data())); - if (!UseLhsDirectly) Map(actualLhsPtr, actualLhs.size()) = actualLhs; + EIGEN_IF_CONSTEXPR(!UseLhsDirectly) { + Map(actualLhsPtr, actualLhs.size()) = actualLhs; + } internal::gemv_static_vector_if static_rhs; ei_declare_aligned_stack_constructed_variable( Scalar, actualRhsPtr, actualRhs.size(), (UseRhsDirectly ? const_cast(actualRhs.data()) : static_rhs.data())); - if (!UseRhsDirectly) Map(actualRhsPtr, actualRhs.size()) = actualRhs; + EIGEN_IF_CONSTEXPR(!UseRhsDirectly) { + Map(actualRhsPtr, actualRhs.size()) = actualRhs; + } selfadjoint_rank1_update< Scalar, Index, StorageOrder, UpLo, LhsBlasTraits::NeedToConjugate && NumTraits::IsComplex, diff --git a/Eigen/src/Core/products/GeneralMatrixVector.h b/Eigen/src/Core/products/GeneralMatrixVector.h index 3877e324b..eb667a845 100644 --- a/Eigen/src/Core/products/GeneralMatrixVector.h +++ b/Eigen/src/Core/products/GeneralMatrixVector.h @@ -255,25 +255,29 @@ general_matrix_vector_product(i, j2, jend, lhs, rhs, res, palpha, pcj); i += ResPacketSize; } - if (HasHalf && i < n_half) { - ResPacketHalf c0 = pzero(ResPacketHalf{}); - for (Index j = j2; j < jend; j += 1) { - RhsPacketHalf b0 = pset1(rhs(j, 0)); - c0 = pcj_half.pmadd(lhs.template load(i + 0, j), b0, c0); + EIGEN_IF_CONSTEXPR(HasHalf) { + if (i < n_half) { + ResPacketHalf c0 = pzero(ResPacketHalf{}); + for (Index j = j2; j < jend; j += 1) { + RhsPacketHalf b0 = pset1(rhs(j, 0)); + c0 = pcj_half.pmadd(lhs.template load(i + 0, j), b0, c0); + } + pstoreu(res + i + ResPacketSizeHalf * 0, + pmadd(c0, palpha_half, ploadu(res + i + ResPacketSizeHalf * 0))); + i += ResPacketSizeHalf; } - pstoreu(res + i + ResPacketSizeHalf * 0, - pmadd(c0, palpha_half, ploadu(res + i + ResPacketSizeHalf * 0))); - i += ResPacketSizeHalf; } - if (HasQuarter && i < n_quarter) { - ResPacketQuarter c0 = pzero(ResPacketQuarter{}); - for (Index j = j2; j < jend; j += 1) { - RhsPacketQuarter b0 = pset1(rhs(j, 0)); - c0 = pcj_quarter.pmadd(lhs.template load(i + 0, j), b0, c0); + EIGEN_IF_CONSTEXPR(HasQuarter) { + if (i < n_quarter) { + ResPacketQuarter c0 = pzero(ResPacketQuarter{}); + for (Index j = j2; j < jend; j += 1) { + RhsPacketQuarter b0 = pset1(rhs(j, 0)); + c0 = pcj_quarter.pmadd(lhs.template load(i + 0, j), b0, c0); + } + pstoreu(res + i + ResPacketSizeQuarter * 0, + pmadd(c0, palpha_quarter, ploadu(res + i + ResPacketSizeQuarter * 0))); + i += ResPacketSizeQuarter; } - pstoreu(res + i + ResPacketSizeQuarter * 0, - pmadd(c0, palpha_quarter, ploadu(res + i + ResPacketSizeQuarter * 0))); - i += ResPacketSizeQuarter; } for (; i < rows; ++i) { ResScalar c0(0); @@ -507,14 +511,14 @@ general_matrix_vector_product(i, j), b0, c0); } ResScalar cc0 = predux(c0); - if (HasHalf) { + EIGEN_IF_CONSTEXPR(HasHalf) { for (Index j = fullColBlockEnd; j < halfColBlockEnd; j += LhsPacketSizeHalf) { RhsPacketHalf b0 = rhs.template load(j, 0); c0_h = pcj_half.pmadd(lhs.template load(i, j), b0, c0_h); } cc0 += predux(c0_h); } - if (HasQuarter) { + EIGEN_IF_CONSTEXPR(HasQuarter) { for (Index j = halfColBlockEnd; j < quarterColBlockEnd; j += LhsPacketSizeQuarter) { RhsPacketQuarter b0 = rhs.template load(j, 0); c0_q = pcj_quarter.pmadd(lhs.template load(i, j), b0, c0_q); @@ -627,7 +631,7 @@ general_matrix_vector_product; ResScalar cc[N] = {}; - if (HasHalf) { + EIGEN_IF_CONSTEXPR(HasHalf) { ResPacketHalf h[N]; Unroll::init_zero(h); for (Index j = 0; j < halfColBlockEnd; j += LhsPacketSizeHalf) { @@ -636,7 +640,7 @@ general_matrix_vector_product= 3 * PacketSize) - for (Index i = 0; i < peeled_mc3; i += 3 * PacketSize) pack<3 * PacketSize>(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR(Pack1 >= 3 * PacketSize) + for (Index i = 0; i < peeled_mc3; i += 3 * PacketSize) pack<3 * PacketSize>(blockA, lhs, cols, i, count); - if (Pack1 >= 2 * PacketSize) - for (Index i = peeled_mc3; i < peeled_mc2; i += 2 * PacketSize) pack<2 * PacketSize>(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR(Pack1 >= 2 * PacketSize) + for (Index i = peeled_mc3; i < peeled_mc2; i += 2 * PacketSize) pack<2 * PacketSize>(blockA, lhs, cols, i, count); - if (Pack1 >= 1 * PacketSize) - for (Index i = peeled_mc2; i < peeled_mc1; i += 1 * PacketSize) pack<1 * PacketSize>(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR(Pack1 >= 1 * PacketSize) + for (Index i = peeled_mc2; i < peeled_mc1; i += 1 * PacketSize) pack<1 * PacketSize>(blockA, lhs, cols, i, count); - if (HasHalf && Pack1 >= HalfPacketSize) - for (Index i = peeled_mc1; i < peeled_mc_half; i += HalfPacketSize) - pack(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR(HasHalf && Pack1 >= HalfPacketSize) + for (Index i = peeled_mc1; i < peeled_mc_half; i += HalfPacketSize) + pack(blockA, lhs, cols, i, count); - if (HasQuarter && Pack1 >= QuarterPacketSize) - for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += QuarterPacketSize) - pack(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR(HasQuarter && Pack1 >= QuarterPacketSize) + for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += QuarterPacketSize) + pack(blockA, lhs, cols, i, count); // do the same with mr==1 for (Index i = peeled_mc_quarter; i < rows; i++) { @@ -112,11 +112,11 @@ struct symm_pack_rhs { for (Index k = k2; k < end_k; k++) { blockB[count + 0] = rhs(k, j2 + 0); blockB[count + 1] = rhs(k, j2 + 1); - if (nr >= 4) { + EIGEN_IF_CONSTEXPR(nr >= 4) { blockB[count + 2] = rhs(k, j2 + 2); blockB[count + 3] = rhs(k, j2 + 3); } - if (nr >= 8) { + EIGEN_IF_CONSTEXPR(nr >= 8) { blockB[count + 4] = rhs(k, j2 + 4); blockB[count + 5] = rhs(k, j2 + 5); blockB[count + 6] = rhs(k, j2 + 6); @@ -128,7 +128,7 @@ struct symm_pack_rhs { // second part: diagonal block Index end8 = nr >= 8 ? (std::min)(k2 + rows, packet_cols8) : k2; - if (nr >= 8) { + EIGEN_IF_CONSTEXPR(nr >= 8) { for (Index j2 = k2; j2 < end8; j2 += 8) { // again we can split vertically in three different parts (transpose, symmetric, normal) // transpose @@ -170,7 +170,7 @@ struct symm_pack_rhs { } } } - if (nr >= 4) { + EIGEN_IF_CONSTEXPR(nr >= 4) { for (Index j2 = end8; j2 < (std::min)(k2 + rows, packet_cols4); j2 += 4) { // again we can split vertically in three different parts (transpose, symmetric, normal) // transpose @@ -206,7 +206,7 @@ struct symm_pack_rhs { } // third part: transposed - if (nr >= 8) { + EIGEN_IF_CONSTEXPR(nr >= 8) { for (Index j2 = k2 + rows; j2 < packet_cols8; j2 += 8) { for (Index k = k2; k < end_k; k++) { blockB[count + 0] = numext::conj(rhs(j2 + 0, k)); @@ -221,7 +221,7 @@ struct symm_pack_rhs { } } } - if (nr >= 4) { + EIGEN_IF_CONSTEXPR(nr >= 4) { for (Index j2 = (std::max)(packet_cols8, k2 + rows); j2 < packet_cols4; j2 += 4) { for (Index k = k2; k < end_k; k++) { blockB[count + 0] = numext::conj(rhs(j2 + 0, k)); diff --git a/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h b/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h index 9873b2516..e5838bf7d 100644 --- a/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h +++ b/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h @@ -57,10 +57,11 @@ struct selfadjoint_matrix_vector_product_symv Specialized> { \ static void run(Index size, const Scalar* lhs, Index lhsStride, const Scalar* _rhs, Scalar* res, Scalar alpha) { \ enum { IsColMajor = StorageOrder == ColMajor }; \ - if (IsColMajor == ConjugateLhs) { \ + EIGEN_IF_CONSTEXPR(IsColMajor == ConjugateLhs) { \ selfadjoint_matrix_vector_product::run(size, lhs, lhsStride, _rhs, res, alpha); \ - } else { \ + } \ + else { \ selfadjoint_matrix_vector_product_symv::run( \ size, lhs, lhsStride, _rhs, res, alpha); \ } \ diff --git a/Eigen/src/Core/products/SelfadjointProduct.h b/Eigen/src/Core/products/SelfadjointProduct.h index fb2e987c0..d486129f4 100644 --- a/Eigen/src/Core/products/SelfadjointProduct.h +++ b/Eigen/src/Core/products/SelfadjointProduct.h @@ -40,7 +40,7 @@ struct selfadjoint_rank1_update Packet ps0 = internal::pset1(s0); Packet ps1 = internal::pset1(s1); - if (UpLo == Lower) { + EIGEN_IF_CONSTEXPR(UpLo == Lower) { Scalar* EIGEN_RESTRICT col0 = mat + stride * j + j; Scalar* EIGEN_RESTRICT col1 = mat + stride * (j + 1) + (j + 1); @@ -71,7 +71,8 @@ struct selfadjoint_rank1_update d0[k] += s0 * cx; d1[k] += s1 * cx; } - } else { + } + else { // UpLo == Upper Scalar* EIGEN_RESTRICT col0 = mat + stride * j; Scalar* EIGEN_RESTRICT col1 = mat + stride * (j + 1); @@ -161,8 +162,9 @@ struct selfadjoint_product_selector { Scalar, actualOtherPtr, other.size(), (UseOtherDirectly ? const_cast(actualOther.data()) : static_other.data())); - if (!UseOtherDirectly) + EIGEN_IF_CONSTEXPR(!UseOtherDirectly) { Map(actualOtherPtr, actualOther.size()) = actualOther; + } selfadjoint_rank1_update< Scalar, Index, StorageOrder, UpLo, OtherBlasTraits::NeedToConjugate && NumTraits::IsComplex, diff --git a/Eigen/src/Core/products/SelfadjointRank2Update.h b/Eigen/src/Core/products/SelfadjointRank2Update.h index 823db85ec..a80d2f227 100644 --- a/Eigen/src/Core/products/SelfadjointRank2Update.h +++ b/Eigen/src/Core/products/SelfadjointRank2Update.h @@ -220,7 +220,7 @@ EIGEN_DEVICE_FUNC SelfAdjointView& SelfAdjointView& SelfAdjointView(actualU.data()) : static_u.data())); - if (!UseUDirectly) { - if (NeedConjU) - Map(uPtr, size) = actualU.conjugate(); - else + EIGEN_IF_CONSTEXPR(!UseUDirectly) { + EIGEN_IF_CONSTEXPR(NeedConjU) { Map(uPtr, size) = actualU.conjugate(); } + else { Map(uPtr, size) = actualU; + } } // Copy v to contiguous buffer, applying conjugation if needed @@ -241,11 +241,11 @@ EIGEN_DEVICE_FUNC SelfAdjointView& SelfAdjointView(actualV.data()) : static_v.data())); - if (!UseVDirectly) { - if (NeedConjV) - Map(vPtr, size) = actualV.conjugate(); - else + EIGEN_IF_CONSTEXPR(!UseVDirectly) { + EIGEN_IF_CONSTEXPR(NeedConjV) { Map(vPtr, size) = actualV.conjugate(); } + else { Map(vPtr, size) = actualV; + } } internal::selfadjoint_rank2_update_selector< diff --git a/Eigen/src/Core/products/TriangularMatrixMatrix.h b/Eigen/src/Core/products/TriangularMatrixMatrix.h index 84e09cfc6..0a3b22b50 100644 --- a/Eigen/src/Core/products/TriangularMatrixMatrix.h +++ b/Eigen/src/Core/products/TriangularMatrixMatrix.h @@ -92,10 +92,9 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix< Matrix triangularBuffer; triangularBuffer.setZero(); - if ((Mode & ZeroDiag) == ZeroDiag) - triangularBuffer.diagonal().setZero(); - else - triangularBuffer.diagonal().setOnes(); + EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag) + triangularBuffer.diagonal().setZero(); + else triangularBuffer.diagonal().setOnes(); gebp_kernel gebp_kernel; gemm_pack_lhs triangularBuffer; triangularBuffer.setZero(); - if ((Mode & ZeroDiag) == ZeroDiag) - triangularBuffer.diagonal().setZero(); - else - triangularBuffer.diagonal().setOnes(); + EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag) + triangularBuffer.diagonal().setZero(); + else triangularBuffer.diagonal().setOnes(); gebp_kernel gebp_kernel; gemm_pack_lhs cols)) { - actual_kc = cols - k2; - k2 = actual_k2 + actual_kc - kc; + EIGEN_IF_CONSTEXPR(IsLower) { + if ((k2 < cols) && (actual_k2 + actual_kc > cols)) { + actual_kc = cols - k2; + k2 = actual_k2 + actual_kc - kc; + } } // remaining size @@ -355,7 +355,7 @@ struct triangular_product_impl { actualAlpha, blocking); // Apply correction if the diagonal is unit and a scalar factor was nested: - if ((Mode & UnitDiag) == UnitDiag) { + EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) { if (LhsIsTriangular && !numext::is_exactly_one(lhs_alpha)) { Index diagSize = (std::min)(lhs.rows(), lhs.cols()); dst.topRows(diagSize) -= ((lhs_alpha - LhsScalar(1)) * a_rhs).topRows(diagSize); diff --git a/Eigen/src/Core/products/TriangularMatrixVector.h b/Eigen/src/Core/products/TriangularMatrixVector.h index 14dacaf99..2791d5eb8 100644 --- a/Eigen/src/Core/products/TriangularMatrixVector.h +++ b/Eigen/src/Core/products/TriangularMatrixVector.h @@ -55,7 +55,7 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product size) { - general_matrix_vector_product::run( - rows, cols - size, LhsMapper(&lhs_[size * lhsStride], lhsStride), RhsMapper(&rhs_[size * rhsIncr], rhsIncr), - res_, resIncr, alpha); + EIGEN_IF_CONSTEXPR(!IsLower) { + if (cols > size) { + general_matrix_vector_product::run( + rows, cols - size, LhsMapper(&lhs_[size * lhsStride], lhsStride), RhsMapper(&rhs_[size * rhsIncr], rhsIncr), + res_, resIncr, alpha); + } } } @@ -181,17 +184,18 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product diagSize) { - general_matrix_vector_product::run( - rows - diagSize, cols, LhsMapper(&lhs_[diagSize * lhsStride], lhsStride), RhsMapper(rhs_, rhsIncr), - &res_[diagSize * resIncr], resIncr, alpha); + EIGEN_IF_CONSTEXPR(IsLower) { + if (rows > diagSize) { + general_matrix_vector_product::run( + rows - diagSize, cols, LhsMapper(&lhs_[diagSize * lhsStride], lhsStride), RhsMapper(rhs_, rhsIncr), + &res_[diagSize * resIncr], resIncr, alpha); + } } } diff --git a/Eigen/src/Core/util/Memory.h b/Eigen/src/Core/util/Memory.h index adb31c3ca..1ba8068c9 100644 --- a/Eigen/src/Core/util/Memory.h +++ b/Eigen/src/Core/util/Memory.h @@ -494,7 +494,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) { if (size == 0) return nullptr; // short-cut. Also fixes Bug 884 check_size_for_overflow(size); T* result = static_cast(conditional_aligned_malloc(sizeof(T) * size)); - if (NumTraits::RequireInitialization) { + EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { EIGEN_TRY { default_construct_elements_of_array(result, size); } EIGEN_CATCH(...) { conditional_aligned_free(result); @@ -506,7 +506,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) { template EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) { - if (NumTraits::RequireInitialization) { + EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { return conditional_aligned_realloc_new(pts, new_size, old_size); } @@ -518,7 +518,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::si template EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T* ptr, std::size_t size) { - if (NumTraits::RequireInitialization) destruct_elements_of_array(ptr, size); + EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) destruct_elements_of_array(ptr, size); conditional_aligned_free(ptr); } @@ -673,10 +673,14 @@ class aligned_stack_memory_handler { **/ EIGEN_DEVICE_FUNC aligned_stack_memory_handler(T* ptr, std::size_t size, bool dealloc) : m_ptr(ptr), m_size(size), m_deallocate(dealloc) { - if (NumTraits::RequireInitialization && m_ptr) Eigen::internal::default_construct_elements_of_array(m_ptr, size); + EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + if (m_ptr) Eigen::internal::default_construct_elements_of_array(m_ptr, size); + } } EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() { - if (NumTraits::RequireInitialization && m_ptr) Eigen::internal::destruct_elements_of_array(m_ptr, m_size); + EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + if (m_ptr) Eigen::internal::destruct_elements_of_array(m_ptr, m_size); + } if (m_deallocate) Eigen::internal::aligned_free(m_ptr); } @@ -714,14 +718,16 @@ struct local_nested_eval_wrapper { : object(ptr == 0 ? reinterpret_cast(Eigen::internal::aligned_malloc(sizeof(Scalar) * xpr.size())) : ptr, xpr.rows(), xpr.cols()), m_deallocate(ptr == 0) { - if (NumTraits::RequireInitialization && object.data()) - Eigen::internal::default_construct_elements_of_array(object.data(), object.size()); + EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + if (object.data()) Eigen::internal::default_construct_elements_of_array(object.data(), object.size()); + } object = xpr; } EIGEN_DEVICE_FUNC ~local_nested_eval_wrapper() { - if (NumTraits::RequireInitialization && object.data()) - Eigen::internal::destruct_elements_of_array(object.data(), object.size()); + EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + if (object.data()) Eigen::internal::destruct_elements_of_array(object.data(), object.size()); + } if (m_deallocate) Eigen::internal::aligned_free(object.data()); } diff --git a/Eigen/src/Geometry/OrthoMethods.h b/Eigen/src/Geometry/OrthoMethods.h index d8f7304e3..ed873b0a9 100644 --- a/Eigen/src/Geometry/OrthoMethods.h +++ b/Eigen/src/Geometry/OrthoMethods.h @@ -157,12 +157,13 @@ VectorwiseOp::cross(const MatrixBase& o typename internal::nested_eval::type vec(other.derived()); CrossReturnType res(_expression().rows(), _expression().cols()); - if (Direction == Vertical) { + EIGEN_IF_CONSTEXPR(Direction == Vertical) { eigen_assert(CrossReturnType::RowsAtCompileTime == 3 && "the matrix must have exactly 3 rows"); res.row(0) = (mat.row(1) * vec.coeff(2) - mat.row(2) * vec.coeff(1)).conjugate(); res.row(1) = (mat.row(2) * vec.coeff(0) - mat.row(0) * vec.coeff(2)).conjugate(); res.row(2) = (mat.row(0) * vec.coeff(1) - mat.row(1) * vec.coeff(0)).conjugate(); - } else { + } + else { eigen_assert(CrossReturnType::ColsAtCompileTime == 3 && "the matrix must have exactly 3 columns"); res.col(0) = (mat.col(1) * vec.coeff(2) - mat.col(2) * vec.coeff(1)).conjugate(); res.col(1) = (mat.col(2) * vec.coeff(0) - mat.col(0) * vec.coeff(2)).conjugate(); diff --git a/Eigen/src/Geometry/Transform.h b/Eigen/src/Geometry/Transform.h index feb28545a..3dc563cb8 100644 --- a/Eigen/src/Geometry/Transform.h +++ b/Eigen/src/Geometry/Transform.h @@ -1429,7 +1429,7 @@ struct transform_left_product_impl { typedef TransformType ResultType; static EIGEN_DEVICE_FUNC ResultType run(const Other& other, const TransformType& tr) { TransformType res; - if (Mode != int(AffineCompact)) res.matrix().row(Dim) = tr.matrix().row(Dim); + EIGEN_IF_CONSTEXPR(Mode != int(AffineCompact)) res.matrix().row(Dim) = tr.matrix().row(Dim); res.matrix().template topRows().noalias() = other * tr.matrix().template topRows(); return res; } diff --git a/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h b/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h index 982f6ae29..7ed53c741 100644 --- a/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h +++ b/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h @@ -142,7 +142,7 @@ class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner LeastSquareDiagonalPreconditioner& factorize(const MatType& mat) { // Compute the inverse squared-norm of each column of mat m_invdiag.resize(mat.cols()); - if (MatType::IsRowMajor) { + EIGEN_IF_CONSTEXPR(MatType::IsRowMajor) { m_invdiag.setZero(); for (Index j = 0; j < mat.outerSize(); ++j) { for (typename MatType::InnerIterator it(mat, j); it; ++it) m_invdiag(it.index()) += numext::abs2(it.value()); @@ -151,7 +151,8 @@ class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner RealScalar sum = numext::real(m_invdiag(j)); m_invdiag(j) = sum > RealScalar(0) ? RealScalar(1) / sum : RealScalar(1); } - } else { + } + else { for (Index j = 0; j < mat.outerSize(); ++j) { RealScalar sum = mat.col(j).squaredNorm(); m_invdiag(j) = sum > RealScalar(0) ? RealScalar(1) / sum : RealScalar(1); diff --git a/Eigen/src/SVD/JacobiSVD.h b/Eigen/src/SVD/JacobiSVD.h index 78bd51a1e..a789487d2 100644 --- a/Eigen/src/SVD/JacobiSVD.h +++ b/Eigen/src/SVD/JacobiSVD.h @@ -841,7 +841,11 @@ JacobiSVD& JacobiSVD::compute_impl(con // For a complex matrix, some diagonal coefficients might note have been // treated by svd_precondition_2x2_block_to_be_real, and the imaginary part // of some diagonal entry might not be null. - if (NumTraits::IsComplex && abs(numext::imag(m_workMatrix.coeff(i, i))) > considerAsZero) { + bool diagonal_has_imaginary_part = false; + EIGEN_IF_CONSTEXPR(NumTraits::IsComplex) { + diagonal_has_imaginary_part = abs(numext::imag(m_workMatrix.coeff(i, i))) > considerAsZero; + } + if (diagonal_has_imaginary_part) { RealScalar a = abs(m_workMatrix.coeff(i, i)); m_singularValues.coeffRef(i) = abs(a); if (computeU()) m_matrixU.col(i) *= m_workMatrix.coeff(i, i) / a; diff --git a/Eigen/src/SVD/SVDBase.h b/Eigen/src/SVD/SVDBase.h index 695829372..d150c0bd8 100644 --- a/Eigen/src/SVD/SVDBase.h +++ b/Eigen/src/SVD/SVDBase.h @@ -424,10 +424,12 @@ bool SVDBase::allocate(Index rows, Index cols, unsigned int computation m_diagSize.setValue(numext::mini(m_rows.value(), m_cols.value())); m_singularValues.resize(m_diagSize.value()); - if (RowsAtCompileTime == Dynamic) + EIGEN_IF_CONSTEXPR(RowsAtCompileTime == Dynamic) { m_matrixU.resize(m_rows.value(), m_computeFullU ? m_rows.value() : m_computeThinU ? m_diagSize.value() : 0); - if (ColsAtCompileTime == Dynamic) + } + EIGEN_IF_CONSTEXPR(ColsAtCompileTime == Dynamic) { m_matrixV.resize(m_cols.value(), m_computeFullV ? m_cols.value() : m_computeThinV ? m_diagSize.value() : 0); + } return false; } diff --git a/Eigen/src/SparseCholesky/SimplicialCholesky.h b/Eigen/src/SparseCholesky/SimplicialCholesky.h index cb40e782e..23fb80abc 100644 --- a/Eigen/src/SparseCholesky/SimplicialCholesky.h +++ b/Eigen/src/SparseCholesky/SimplicialCholesky.h @@ -867,7 +867,7 @@ void SimplicialCholeskyBase::ordering(const MatrixType& a, ConstCholMat const Index size = a.rows(); pmat = ≈ // Note that ordering methods compute the inverse permutation - if (!std::is_same >::value) { + EIGEN_IF_CONSTEXPR((!std::is_same >::value)) { { CholMatrixType C; constexpr bool kUseAMDFastPath = std::is_same >::value; @@ -882,15 +882,16 @@ void SimplicialCholeskyBase::ordering(const MatrixType& a, ConstCholMat ap.resize(size, size); internal::permute_symm_to_symm(a, ap, m_P.indices().data()); - } else { + } + else { m_Pinv.resize(0); m_P.resize(0); - if (int(UpLo) == int(Lower) || MatrixType::IsRowMajor) { + EIGEN_IF_CONSTEXPR(int(UpLo) == int(Lower) || MatrixType::IsRowMajor) { // we have to transpose the lower part to to the upper one ap.resize(size, size); internal::permute_symm_to_symm(a, ap, NULL); - } else - internal::simplicial_cholesky_grab_input::run(a, pmat, ap); + } + else internal::simplicial_cholesky_grab_input::run(a, pmat, ap); } } diff --git a/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h b/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h index ee355ee29..c2d6a3ce7 100644 --- a/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h +++ b/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h @@ -375,7 +375,12 @@ void SimplicialCholeskyBase::factorize_preordered(const CholMatrixType& } else { Index p = Lp[k] + nonZerosPerCol[k]++; Li[p] = k; /* store L(k,k) = sqrt (d) in column k */ - if (NonHermitian ? d == RealScalar(0) : numext::real(d) <= RealScalar(0)) { + bool failed; + EIGEN_IF_CONSTEXPR(NonHermitian) { failed = d == RealScalar(0); } + else { + failed = numext::real(d) <= RealScalar(0); + } + if (failed) { ok = false; /* failure, matrix is not positive definite */ break; } diff --git a/Eigen/src/SparseCore/SparseAssign.h b/Eigen/src/SparseCore/SparseAssign.h index 686abeae1..1f92e491c 100644 --- a/Eigen/src/SparseCore/SparseAssign.h +++ b/Eigen/src/SparseCore/SparseAssign.h @@ -144,8 +144,9 @@ struct Assignment { template struct Assignment { static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { - if (std::is_same>::value) - dst.setZero(); + EIGEN_IF_CONSTEXPR( + (std::is_same>::value)) + dst.setZero(); internal::evaluator srcEval(src); resize_if_allowed(dst, src, func); diff --git a/Eigen/src/SparseCore/SparseBlock.h b/Eigen/src/SparseCore/SparseBlock.h index f17632e1d..c5e2bf0f3 100644 --- a/Eigen/src/SparseCore/SparseBlock.h +++ b/Eigen/src/SparseCore/SparseBlock.h @@ -204,10 +204,11 @@ class sparse_matrix_block_impl : public SparseCompressedBase(tmp.innerVector(k).nonZeros()); diff --git a/Eigen/src/SparseCore/SparseDenseProduct.h b/Eigen/src/SparseCore/SparseDenseProduct.h index 3bd724b5a..1c6de14b5 100644 --- a/Eigen/src/SparseCore/SparseDenseProduct.h +++ b/Eigen/src/SparseCore/SparseDenseProduct.h @@ -198,33 +198,35 @@ struct sparse_time_dense_product_impl reports innerStride()==1 but is actually RowMajor, so check both. - if (!(Res::Flags & RowMajorBit) && res.innerStride() == 1) { - for (Index c = 0; c < rhs.cols(); ++c) { - typename Res::Scalar* y = res.data() + c * res.outerStride(); - for (Index j = 0; j < lhs.outerSize(); ++j) { - typename ScalarBinaryOpTraits::ReturnType rhs_j(alpha * rhs.coeff(j, c)); - const Index start = outer ? outer[j] : 0; - const Index end = innerNnz ? start + innerNnz[j] : (outer ? outer[j + 1] : mat.nonZeros()); - Index k = start; - // 4-way unrolled scatter-add (no SIMD: writes are scattered) - for (; k + 3 < end; k += 4) { - y[inds[k]] += vals[k] * rhs_j; - y[inds[k + 1]] += vals[k + 1] * rhs_j; - y[inds[k + 2]] += vals[k + 2] * rhs_j; - y[inds[k + 3]] += vals[k + 3] * rhs_j; + EIGEN_IF_CONSTEXPR(!(Res::Flags & RowMajorBit)) { + if (res.innerStride() == 1) { + for (Index c = 0; c < rhs.cols(); ++c) { + typename Res::Scalar* y = res.data() + c * res.outerStride(); + for (Index j = 0; j < lhs.outerSize(); ++j) { + typename ScalarBinaryOpTraits::ReturnType rhs_j(alpha * rhs.coeff(j, c)); + const Index start = outer ? outer[j] : 0; + const Index end = innerNnz ? start + innerNnz[j] : (outer ? outer[j + 1] : mat.nonZeros()); + Index k = start; + // 4-way unrolled scatter-add (no SIMD: writes are scattered) + for (; k + 3 < end; k += 4) { + y[inds[k]] += vals[k] * rhs_j; + y[inds[k + 1]] += vals[k + 1] * rhs_j; + y[inds[k + 2]] += vals[k + 2] * rhs_j; + y[inds[k + 3]] += vals[k + 3] * rhs_j; + } + for (; k < end; ++k) y[inds[k]] += vals[k] * rhs_j; } - for (; k < end; ++k) y[inds[k]] += vals[k] * rhs_j; } + return; } - } else { - // Non-unit result stride: use coeffRef() for result access - for (Index c = 0; c < rhs.cols(); ++c) { - for (Index j = 0; j < lhs.outerSize(); ++j) { - typename ScalarBinaryOpTraits::ReturnType rhs_j(alpha * rhs.coeff(j, c)); - const Index start = outer ? outer[j] : 0; - const Index end = innerNnz ? start + innerNnz[j] : (outer ? outer[j + 1] : mat.nonZeros()); - for (Index k = start; k < end; ++k) res.coeffRef(inds[k], c) += vals[k] * rhs_j; - } + } + // Non-unit result stride: use coeffRef() for result access + for (Index c = 0; c < rhs.cols(); ++c) { + for (Index j = 0; j < lhs.outerSize(); ++j) { + typename ScalarBinaryOpTraits::ReturnType rhs_j(alpha * rhs.coeff(j, c)); + const Index start = outer ? outer[j] : 0; + const Index end = innerNnz ? start + innerNnz[j] : (outer ? outer[j + 1] : mat.nonZeros()); + for (Index k = start; k < end; ++k) res.coeffRef(inds[k], c) += vals[k] * rhs_j; } } } diff --git a/Eigen/src/SparseCore/SparseDiagonalProduct.h b/Eigen/src/SparseCore/SparseDiagonalProduct.h index 0eec0277a..cee2aeca3 100644 --- a/Eigen/src/SparseCore/SparseDiagonalProduct.h +++ b/Eigen/src/SparseCore/SparseDiagonalProduct.h @@ -144,10 +144,10 @@ struct sparse_selfadjoint_diagonal_product_impl { const StorageIndex outer = outerIndex(row, col); const Index k = count[outer]++; dest.innerIndexPtr()[k] = innerIndex(row, col); - if (ProductOrder == OnTheLeft) - dest.valuePtr()[k] = diagonal.coeff(row) * coeff; - else + EIGEN_IF_CONSTEXPR(ProductOrder == OnTheLeft) { dest.valuePtr()[k] = diagonal.coeff(row) * coeff; } + else { dest.valuePtr()[k] = coeff * diagonal.coeff(col); + } } }; diff --git a/Eigen/src/SparseCore/SparseMatrixBase.h b/Eigen/src/SparseCore/SparseMatrixBase.h index e96c64036..7484f8bbc 100644 --- a/Eigen/src/SparseCore/SparseMatrixBase.h +++ b/Eigen/src/SparseCore/SparseMatrixBase.h @@ -228,7 +228,7 @@ class SparseMatrixBase : public EigenBase { using Nested = typename Derived::Nested; using NestedCleaned = internal::remove_all_t; - if (Flags & RowMajorBit) { + EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { Nested nm(m.derived()); internal::evaluator thisEval(nm); @@ -268,7 +268,8 @@ class SparseMatrixBase : public EigenBase { } s << std::endl; } - } else { + } + else { Nested nm(m.derived()); internal::evaluator thisEval(nm); if (m.cols() == 1) { diff --git a/Eigen/src/SparseCore/SparseRef.h b/Eigen/src/SparseCore/SparseRef.h index 78cf92f71..995499357 100644 --- a/Eigen/src/SparseCore/SparseRef.h +++ b/Eigen/src/SparseCore/SparseRef.h @@ -90,16 +90,18 @@ class SparseRefBase : public SparseMapBase { protected: template void construct(Expression& expr) { - if (Expression::IsVectorAtCompileTime) { + EIGEN_IF_CONSTEXPR(Expression::IsVectorAtCompileTime) { const Index offset = expr.outerIndexPtr() ? expr.outerIndexPtr()[0] : 0; auto inner_index_ptr = expr.innerIndexPtr(); auto value_ptr = expr.valuePtr(); if (inner_index_ptr) inner_index_ptr += offset; if (value_ptr) value_ptr += offset; internal::construct_at(this, expr.size(), expr.nonZeros(), inner_index_ptr, value_ptr); - } else if (expr.outerIndexPtr() == 0) { + } + else if (expr.outerIndexPtr() == 0) { internal::construct_at(this, expr.size(), expr.nonZeros(), expr.innerIndexPtr(), expr.valuePtr()); - } else { + } + else { internal::construct_at(this, expr.rows(), expr.cols(), expr.nonZeros(), expr.outerIndexPtr(), expr.innerIndexPtr(), expr.valuePtr(), expr.innerNonZeroPtr()); } diff --git a/Eigen/src/SparseCore/SparseSelfAdjointView.h b/Eigen/src/SparseCore/SparseSelfAdjointView.h index d62cdd486..bda6e91c7 100644 --- a/Eigen/src/SparseCore/SparseSelfAdjointView.h +++ b/Eigen/src/SparseCore/SparseSelfAdjointView.h @@ -335,7 +335,7 @@ inline void sparse_selfadjoint_time_dense_product(const SparseLhsType& lhs, cons typename DenseResType::Scalar res_j(0); for (; (ProcessFirstHalf ? i && i.index() < j : i); ++i) { LhsScalar lhs_ij = i.value(); - if (!LhsIsRowMajor) lhs_ij = numext::conj(lhs_ij); + EIGEN_IF_CONSTEXPR(!LhsIsRowMajor) { lhs_ij = numext::conj(lhs_ij); } res_j += lhs_ij * rhs.coeff(i.index(), k); res(i.index(), k) += numext::conj(lhs_ij) * rhs_j; } @@ -456,10 +456,9 @@ void permute_symm_to_fullsymm( Index r = it.row(); Index c = it.col(); Index ip = perm ? perm[i] : i; - if (Mode == int(Upper | Lower)) - count[StorageOrderMatch ? jp : ip]++; - else if (r == c) - count[ip]++; + EIGEN_IF_CONSTEXPR(Mode == int(Upper | Lower)) + count[StorageOrderMatch ? jp : ip]++; + else if (r == c) count[ip]++; else if ((Mode == Lower && r > c) || (Mode == Upper && r < c)) { count[ip]++; count[jp]++; @@ -484,16 +483,18 @@ void permute_symm_to_fullsymm( StorageIndex jp = perm ? perm[j] : j; StorageIndex ip = perm ? perm[i] : i; - if (Mode == int(Upper | Lower)) { + EIGEN_IF_CONSTEXPR(Mode == int(Upper | Lower)) { Index k = count[StorageOrderMatch ? jp : ip]++; dest.innerIndexPtr()[k] = StorageOrderMatch ? ip : jp; dest.valuePtr()[k] = it.value(); - } else if (r == c) { + } + else if (r == c) { Index k = count[ip]++; dest.innerIndexPtr()[k] = ip; dest.valuePtr()[k] = it.value(); - } else if (((Mode & Lower) == Lower && r > c) || ((Mode & Upper) == Upper && r < c)) { - if (!StorageOrderMatch) std::swap(ip, jp); + } + else if (((Mode & Lower) == Lower && r > c) || ((Mode & Upper) == Upper && r < c)) { + EIGEN_IF_CONSTEXPR(!StorageOrderMatch) std::swap(ip, jp); Index k = count[jp]++; dest.innerIndexPtr()[k] = ip; dest.valuePtr()[k] = it.value(); @@ -555,7 +556,7 @@ void permute_symm_to_symm(const MatrixType& mat, Index k = count[int(DstMode) == int(Lower) ? (std::min)(ip, jp) : (std::max)(ip, jp)]++; dest.innerIndexPtr()[k] = int(DstMode) == int(Lower) ? (std::max)(ip, jp) : (std::min)(ip, jp); - if (!StorageOrderMatch) std::swap(ip, jp); + EIGEN_IF_CONSTEXPR(!StorageOrderMatch) std::swap(ip, jp); if (((int(DstMode) == int(Lower) && ip < jp) || (int(DstMode) == int(Upper) && ip > jp))) dest.valuePtr()[k] = (NonHermitian ? it.value() : numext::conj(it.value())); else diff --git a/Eigen/src/SparseCore/SparseSparseProductWithPruning.h b/Eigen/src/SparseCore/SparseSparseProductWithPruning.h index 13bb5d900..bb4022bdb 100644 --- a/Eigen/src/SparseCore/SparseSparseProductWithPruning.h +++ b/Eigen/src/SparseCore/SparseSparseProductWithPruning.h @@ -38,10 +38,10 @@ static void sparse_sparse_product_with_pruning_impl(const Lhs& lhs, const Rhs& r AmbiVector tempVector(rows); // mimics a resizeByInnerOuter: - if (ResultType::IsRowMajor) - res.resize(cols, rows); - else + EIGEN_IF_CONSTEXPR(ResultType::IsRowMajor) { res.resize(cols, rows); } + else { res.resize(rows, cols); + } evaluator lhsEval(lhs); evaluator rhsEval(rhs); diff --git a/Eigen/src/SparseCore/SparseTriangularView.h b/Eigen/src/SparseCore/SparseTriangularView.h index 0b2289ae4..b595b26cb 100644 --- a/Eigen/src/SparseCore/SparseTriangularView.h +++ b/Eigen/src/SparseCore/SparseTriangularView.h @@ -105,23 +105,30 @@ struct unary_evaluator, IteratorBased> : evaluator : Base(xprEval.m_argImpl, outer), m_returnOne(false), m_containsDiag(Base::outer() < xprEval.m_arg.innerSize()) { - if (SkipFirst) { + EIGEN_IF_CONSTEXPR(SkipFirst) { while ((*this) && ((HasUnitDiag || SkipDiag) ? this->index() <= outer : this->index() < outer)) Base::operator++(); - if (HasUnitDiag) m_returnOne = m_containsDiag; - } else if (HasUnitDiag && ((!Base::operator bool()) || Base::index() >= Base::outer())) { - if ((!SkipFirst) && Base::operator bool()) Base::operator++(); - m_returnOne = m_containsDiag; + EIGEN_IF_CONSTEXPR(HasUnitDiag) m_returnOne = m_containsDiag; + } + else EIGEN_IF_CONSTEXPR(HasUnitDiag) { + if ((!Base::operator bool()) || Base::index() >= Base::outer()) { + if (Base::operator bool()) Base::operator++(); + m_returnOne = m_containsDiag; + } } } EIGEN_STRONG_INLINE InnerIterator& operator++() { - if (HasUnitDiag && m_returnOne) - m_returnOne = false; - else { - Base::operator++(); - if (HasUnitDiag && (!SkipFirst) && ((!Base::operator bool()) || Base::index() >= Base::outer())) { - if ((!SkipFirst) && Base::operator bool()) Base::operator++(); + EIGEN_IF_CONSTEXPR(HasUnitDiag) { + if (m_returnOne) { + m_returnOne = false; + return *this; + } + } + Base::operator++(); + EIGEN_IF_CONSTEXPR(HasUnitDiag && !SkipFirst) { + if ((!Base::operator bool()) || Base::index() >= Base::outer()) { + if (Base::operator bool()) Base::operator++(); m_returnOne = m_containsDiag; } } @@ -129,30 +136,31 @@ struct unary_evaluator, IteratorBased> : evaluator } EIGEN_STRONG_INLINE operator bool() const { - if (HasUnitDiag && m_returnOne) return true; - if (SkipFirst) - return Base::operator bool(); + EIGEN_IF_CONSTEXPR(HasUnitDiag) { + if (m_returnOne) return true; + } + EIGEN_IF_CONSTEXPR(SkipFirst) { return Base::operator bool(); } else { - if (SkipDiag) - return (Base::operator bool() && this->index() < this->outer()); - else + EIGEN_IF_CONSTEXPR(SkipDiag) { return (Base::operator bool() && this->index() < this->outer()); } + else { return (Base::operator bool() && this->index() <= this->outer()); + } } } inline Index row() const { return (ArgType::Flags & RowMajorBit ? Base::outer() : this->index()); } inline Index col() const { return (ArgType::Flags & RowMajorBit ? this->index() : Base::outer()); } inline StorageIndex index() const { - if (HasUnitDiag && m_returnOne) - return internal::convert_index(Base::outer()); - else - return Base::index(); + EIGEN_IF_CONSTEXPR(HasUnitDiag) { + if (m_returnOne) return internal::convert_index(Base::outer()); + } + return Base::index(); } inline Scalar value() const { - if (HasUnitDiag && m_returnOne) - return Scalar(1); - else - return Base::value(); + EIGEN_IF_CONSTEXPR(HasUnitDiag) { + if (m_returnOne) return Scalar(1); + } + return Base::value(); } protected: diff --git a/Eigen/src/SparseCore/SparsityPatternRef.h b/Eigen/src/SparseCore/SparsityPatternRef.h index 497682847..d9684a765 100644 --- a/Eigen/src/SparseCore/SparsityPatternRef.h +++ b/Eigen/src/SparseCore/SparsityPatternRef.h @@ -382,9 +382,8 @@ void materialize_selfadjoint_pattern(const SparsityPatternRef& A, for (Index j = 0; j < n; ++j) { const StorageIndex* a_col = A.inner + A.outer[j]; const Index a_nz = A.nonZeros(j); - if (IsLower) { - a_split(j) = std::lower_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col; - } else { + EIGEN_IF_CONSTEXPR(IsLower) { a_split(j) = std::lower_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col; } + else { a_split(j) = std::upper_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col; } } diff --git a/Eigen/src/SparseCore/TriangularSolver.h b/Eigen/src/SparseCore/TriangularSolver.h index b68d99711..2c8d0af0d 100644 --- a/Eigen/src/SparseCore/TriangularSolver.h +++ b/Eigen/src/SparseCore/TriangularSolver.h @@ -44,8 +44,8 @@ struct sparse_solve_triangular_selector { if (lastIndex == i) break; tmp = numext::madd(-lastVal, other.coeff(lastIndex, col), tmp); } - if (Mode & UnitDiag) - other.coeffRef(i, col) = tmp; + EIGEN_IF_CONSTEXPR(Mode & UnitDiag) + other.coeffRef(i, col) = tmp; else { eigen_assert(lastIndex == i); other.coeffRef(i, col) = tmp / lastVal; @@ -69,20 +69,19 @@ struct sparse_solve_triangular_selector { Scalar l_ii(0); LhsIterator it(lhsEval, i); while (it && it.index() < i) ++it; - if (!(Mode & UnitDiag)) { + EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) { eigen_assert(it && it.index() == i); l_ii = it.value(); ++it; - } else if (it && it.index() == i) - ++it; + } + else if (it && it.index() == i)++ it; for (; it; ++it) { tmp = numext::madd(-it.value(), other.coeff(it.index(), col), tmp); } - if (Mode & UnitDiag) - other.coeffRef(i, col) = tmp; - else - other.coeffRef(i, col) = tmp / l_ii; + EIGEN_IF_CONSTEXPR(Mode & UnitDiag) + other.coeffRef(i, col) = tmp; + else other.coeffRef(i, col) = tmp / l_ii; } } } @@ -103,7 +102,7 @@ struct sparse_solve_triangular_selector { { LhsIterator it(lhsEval, i); while (it && it.index() < i) ++it; - if (!(Mode & UnitDiag)) { + EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) { eigen_assert(it && it.index() == i); tmp /= it.value(); } @@ -130,7 +129,7 @@ struct sparse_solve_triangular_selector { Scalar& tmp = other.coeffRef(i, col); if (!numext::is_exactly_zero(tmp)) // optimization when other is actually sparse { - if (!(Mode & UnitDiag)) { + EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) { // TODO: replace this with a binary search. make sure the binary search is safe for partially sorted // elements LhsIterator it(lhsEval, i); @@ -211,20 +210,21 @@ struct sparse_solve_triangular_sparse_selector { if (!numext::is_exactly_zero(ci)) { // find typename Lhs::InnerIterator it(lhs, i); - if (!(Mode & UnitDiag)) { - if (IsLower) { + EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) { + EIGEN_IF_CONSTEXPR(IsLower) { eigen_assert(it.index() == i); ci /= it.value(); - } else - ci /= lhs.coeff(i, i); + } + else ci /= lhs.coeff(i, i); } tempVector.restart(); - if (IsLower) { + EIGEN_IF_CONSTEXPR(IsLower) { if (it.index() == i) ++it; for (; it; ++it) { tempVector.coeffRef(it.index()) = numext::madd(-ci, it.value(), tempVector.coeffRef(it.index())); } - } else { + } + else { for (; it && it.index() < i; ++it) { tempVector.coeffRef(it.index()) = numext::madd(-ci, it.value(), tempVector.coeffRef(it.index())); } diff --git a/Eigen/src/SuperLUSupport/SuperLUSupport.h b/Eigen/src/SuperLUSupport/SuperLUSupport.h index 38ab871a0..3cc0b1b4e 100644 --- a/Eigen/src/SuperLUSupport/SuperLUSupport.h +++ b/Eigen/src/SuperLUSupport/SuperLUSupport.h @@ -155,14 +155,11 @@ struct SluMatrix : SuperMatrix { template void setScalarType() { - if (std::is_same::value) - Dtype = SLU_S; - else if (std::is_same::value) - Dtype = SLU_D; - else if (std::is_same >::value) - Dtype = SLU_C; - else if (std::is_same >::value) - Dtype = SLU_Z; + EIGEN_IF_CONSTEXPR((std::is_same::value)) + Dtype = SLU_S; + else EIGEN_IF_CONSTEXPR((std::is_same::value)) Dtype = SLU_D; + else EIGEN_IF_CONSTEXPR((std::is_same >::value)) Dtype = SLU_C; + else EIGEN_IF_CONSTEXPR((std::is_same >::value)) Dtype = SLU_Z; else { eigen_assert(false && "Scalar type not supported by SuperLU"); } @@ -190,11 +187,12 @@ struct SluMatrix : SuperMatrix { static SluMatrix Map(SparseMatrixBase &a_mat) { MatrixType &mat(a_mat.derived()); SluMatrix res; - if ((MatrixType::Flags & RowMajorBit) == RowMajorBit) { + EIGEN_IF_CONSTEXPR((MatrixType::Flags & RowMajorBit) == RowMajorBit) { res.setStorageType(SLU_NR); res.nrow = internal::convert_index(mat.cols()); res.ncol = internal::convert_index(mat.rows()); - } else { + } + else { res.setStorageType(SLU_NC); res.nrow = internal::convert_index(mat.rows()); res.ncol = internal::convert_index(mat.cols()); @@ -210,8 +208,8 @@ struct SluMatrix : SuperMatrix { res.setScalarType(); // FIXME: the following type mapping is approximate. - if (int(MatrixType::Flags) & int(Upper)) res.Mtype = SLU_TRU; - if (int(MatrixType::Flags) & int(Lower)) res.Mtype = SLU_TRL; + EIGEN_IF_CONSTEXPR(int(MatrixType::Flags) & int(Upper)) res.Mtype = SLU_TRU; + EIGEN_IF_CONSTEXPR(int(MatrixType::Flags) & int(Lower)) res.Mtype = SLU_TRL; eigen_assert(((int(MatrixType::Flags) & int(SelfAdjoint)) == 0) && "SelfAdjoint matrix shape not supported by SuperLU"); @@ -241,11 +239,12 @@ template struct SluMatrixMapHelper > { typedef Derived MatrixType; static void run(MatrixType &mat, SluMatrix &res) { - if ((MatrixType::Flags & RowMajorBit) == RowMajorBit) { + EIGEN_IF_CONSTEXPR((MatrixType::Flags & RowMajorBit) == RowMajorBit) { res.setStorageType(SLU_NR); res.nrow = mat.cols(); res.ncol = mat.rows(); - } else { + } + else { res.setStorageType(SLU_NC); res.nrow = mat.rows(); res.ncol = mat.cols(); @@ -261,8 +260,8 @@ struct SluMatrixMapHelper > { res.setScalarType(); // FIXME: the following type mapping is approximate. - if (MatrixType::Flags & Upper) res.Mtype = SLU_TRU; - if (MatrixType::Flags & Lower) res.Mtype = SLU_TRL; + EIGEN_IF_CONSTEXPR(MatrixType::Flags & Upper) res.Mtype = SLU_TRU; + EIGEN_IF_CONSTEXPR(MatrixType::Flags & Lower) res.Mtype = SLU_TRL; eigen_assert(((MatrixType::Flags & SelfAdjoint) == 0) && "SelfAdjoint matrix shape not supported by SuperLU"); } diff --git a/Eigen/src/ThreadPool/RunQueue.h b/Eigen/src/ThreadPool/RunQueue.h index 02419ed47..7b73f5203 100644 --- a/Eigen/src/ThreadPool/RunQueue.h +++ b/Eigen/src/ThreadPool/RunQueue.h @@ -197,9 +197,8 @@ class RunQueue { std::atomic_thread_fence(std::memory_order_acquire); continue; } - if (NeedSizeEstimate) { - return CalculateSize(front, back); - } else { + EIGEN_IF_CONSTEXPR(NeedSizeEstimate) { return CalculateSize(front, back); } + else { // This value will be 0 if the queue is empty, and undefined otherwise. unsigned maybe_zero = ((front ^ back) & kMask2); // Queue size estimate must agree with maybe zero check on the queue diff --git a/blas/BandTriangularSolver.h b/blas/BandTriangularSolver.h index fae6d8931..dcca68cad 100644 --- a/blas/BandTriangularSolver.h +++ b/blas/BandTriangularSolver.h @@ -45,7 +45,7 @@ struct band_solve_triangular_selector 0) other.col(col).segment(IsLower ? i + 1 : i - actual_k, actual_k) -= diff --git a/blas/PackedTriangularMatrixVector.h b/blas/PackedTriangularMatrixVector.h index a7d4ef68b..0e7ac545b 100644 --- a/blas/PackedTriangularMatrixVector.h +++ b/blas/PackedTriangularMatrixVector.h @@ -38,9 +38,7 @@ struct packed_triangular_matrix_vector_product 0)) { ResMap(res + (IsLower ? s + i : 0), r) += alpha * cj(rhs[i]) * ConjLhsType(LhsMap(lhs + s, r)); } - if (HasUnitDiag) { - res[i] += alpha * cj(rhs[i]); - } + EIGEN_IF_CONSTEXPR(HasUnitDiag) { res[i] += alpha * cj(rhs[i]); } lhs += IsLower ? size - i : i + 1; } }; @@ -69,9 +67,7 @@ struct packed_triangular_matrix_vector_product >(rhs + (IsLower ? 0 : i + 1), pi))) .sum(); - if (!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? i : 0]); + EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? i : 0]); IsLower ? lhs += pi + 1 : lhs -= pi + 2; } } @@ -53,7 +53,7 @@ struct packed_triangular_solve_vector 0) Map >(rhs + (IsLower ? i + 1 : 0), r) -= rhs[i] * ConjLhsType(LhsMap(lhs + (IsLower ? 1 : 0), r)); diff --git a/test/array_cwise.cpp b/test/array_cwise.cpp index 1cf36c218..379ce4af1 100644 --- a/test/array_cwise.cpp +++ b/test/array_cwise.cpp @@ -1293,8 +1293,8 @@ struct cast_test_impl { static constexpr int MaxPacketSize = internal::plain_enum_max(SrcPacketSize, DstPacketSize); static void run() { - const Index testRows = RowsAtCompileTime == Dynamic ? ((10 * MaxPacketSize) + 1) : RowsAtCompileTime; - const Index testCols = ColsAtCompileTime == Dynamic ? ((10 * MaxPacketSize) + 1) : ColsAtCompileTime; + constexpr Index testRows = RowsAtCompileTime == Dynamic ? ((10 * MaxPacketSize) + 1) : RowsAtCompileTime; + constexpr Index testCols = ColsAtCompileTime == Dynamic ? ((10 * MaxPacketSize) + 1) : ColsAtCompileTime; const Index testSize = testRows * testCols; const Index minTestSize = 100; const Index repeats = numext::div_ceil(minTestSize, testSize); diff --git a/test/dense_storage.cpp b/test/dense_storage.cpp index 65e47f493..898ed898e 100644 --- a/test/dense_storage.cpp +++ b/test/dense_storage.cpp @@ -203,12 +203,12 @@ void plaintype_tests() { constexpr int ColsAtCompileTime = PlainType::ColsAtCompileTime; constexpr int MaxRowsAtCompileTime = PlainType::MaxRowsAtCompileTime; constexpr int MaxColsAtCompileTime = PlainType::MaxColsAtCompileTime; - const Index expectedDefaultRows = RowsAtCompileTime == Dynamic ? 0 : RowsAtCompileTime; - const Index expectedDefaultCols = ColsAtCompileTime == Dynamic ? 0 : ColsAtCompileTime; - const Index minRows = RowsAtCompileTime == Dynamic ? 0 : RowsAtCompileTime; - const Index minCols = ColsAtCompileTime == Dynamic ? 0 : ColsAtCompileTime; - const Index maxRows = MaxRowsAtCompileTime == Dynamic ? 100 : MaxRowsAtCompileTime; - const Index maxCols = MaxColsAtCompileTime == Dynamic ? 100 : MaxColsAtCompileTime; + constexpr Index expectedDefaultRows = RowsAtCompileTime == Dynamic ? 0 : RowsAtCompileTime; + constexpr Index expectedDefaultCols = ColsAtCompileTime == Dynamic ? 0 : ColsAtCompileTime; + constexpr Index minRows = RowsAtCompileTime == Dynamic ? 0 : RowsAtCompileTime; + constexpr Index minCols = ColsAtCompileTime == Dynamic ? 0 : ColsAtCompileTime; + constexpr Index maxRows = MaxRowsAtCompileTime == Dynamic ? 100 : MaxRowsAtCompileTime; + constexpr Index maxCols = MaxColsAtCompileTime == Dynamic ? 100 : MaxColsAtCompileTime; const Index rows = internal::random(minRows, maxRows); const Index cols = internal::random(minCols, maxCols); // default construction @@ -283,4 +283,4 @@ EIGEN_DECLARE_TEST(dense_storage) { } } -#undef EIGEN_TESTING_PLAINOBJECT_CTOR \ No newline at end of file +#undef EIGEN_TESTING_PLAINOBJECT_CTOR diff --git a/test/packetmath.cpp b/test/packetmath.cpp index 8f4abc7be..fae4fc7c6 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp @@ -572,7 +572,7 @@ void packetmath() { std::cerr << "=== Testing packet of type '" << typeid(Packet).name() << "' and scalar type '" << typeid(Scalar).name() << "' and size '" << PacketSize << "' ===\n"; - const int max_size = PacketSize > 4 ? PacketSize : 4; + constexpr int max_size = PacketSize > 4 ? PacketSize : 4; const int size = PacketSize * max_size; EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data1[size]; EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar data2[size]; diff --git a/test/product_extra.cpp b/test/product_extra.cpp index 339444890..06df124dc 100644 --- a/test/product_extra.cpp +++ b/test/product_extra.cpp @@ -130,7 +130,7 @@ template void zero_sized_objects(const MatrixType& m) { typedef typename MatrixType::Scalar Scalar; const int PacketSize = internal::packet_traits::size; - const int PacketSize1 = PacketSize > 1 ? PacketSize - 1 : 1; + constexpr int PacketSize1 = PacketSize > 1 ? PacketSize - 1 : 1; Index rows = m.rows(); Index cols = m.cols(); diff --git a/test/product_trmm.cpp b/test/product_trmm.cpp index faf46fa8b..89859f44c 100644 --- a/test/product_trmm.cpp +++ b/test/product_trmm.cpp @@ -13,7 +13,7 @@ template int get_random_size() { const int factor = NumTraits::ReadCost; - const int max_test_size = EIGEN_TEST_MAX_SIZE > 2 * factor ? EIGEN_TEST_MAX_SIZE / factor : EIGEN_TEST_MAX_SIZE; + constexpr int max_test_size = EIGEN_TEST_MAX_SIZE > 2 * factor ? EIGEN_TEST_MAX_SIZE / factor : EIGEN_TEST_MAX_SIZE; return internal::random(1, max_test_size); } diff --git a/test/redux.cpp b/test/redux.cpp index d5ae402ce..432235906 100644 --- a/test/redux.cpp +++ b/test/redux.cpp @@ -73,8 +73,8 @@ void matrixRedux(const MatrixType& m) { VERIFY_IS_APPROX(m1.block(r0, c0, r1, c1).real().maxCoeff(), m1.block(r0, c0, r1, c1).real().eval().maxCoeff()); // regression for bug 1090 - const int R1 = MatrixType::RowsAtCompileTime >= 2 ? MatrixType::RowsAtCompileTime / 2 : 6; - const int C1 = MatrixType::ColsAtCompileTime >= 2 ? MatrixType::ColsAtCompileTime / 2 : 6; + constexpr int R1 = MatrixType::RowsAtCompileTime >= 2 ? MatrixType::RowsAtCompileTime / 2 : 6; + constexpr int C1 = MatrixType::ColsAtCompileTime >= 2 ? MatrixType::ColsAtCompileTime / 2 : 6; if (R1 <= rows - r0 && C1 <= cols - c0) { VERIFY_IS_APPROX((m1.template block(r0, c0).sum()), m1.block(r0, c0, R1, C1).sum()); } diff --git a/test/ulp_accuracy/ulp_accuracy.cpp b/test/ulp_accuracy/ulp_accuracy.cpp index 24627f60e..be6cba928 100644 --- a/test/ulp_accuracy/ulp_accuracy.cpp +++ b/test/ulp_accuracy/ulp_accuracy.cpp @@ -405,8 +405,8 @@ struct Options { template static int run_test(const Options& opts) { - const int kDigits = std::is_same::value ? 9 : 17; - const char* kTypeName = std::is_same::value ? "float" : "double"; + constexpr int kDigits = std::is_same::value ? 9 : 17; + constexpr const char* kTypeName = std::is_same::value ? "float" : "double"; auto table = build_func_table(); diff --git a/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h b/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h index 6fb64720f..914d682a5 100644 --- a/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h +++ b/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h @@ -94,10 +94,10 @@ template struct TripletComp { typedef typename Iterator::value_type Triplet; bool operator()(const Triplet& a, const Triplet& b) { - if (IsColMajor) - return ((a.col() == b.col() && a.row() < b.row()) || (a.col() < b.col())); - else + EIGEN_IF_CONSTEXPR(IsColMajor) { return ((a.col() == b.col() && a.row() < b.row()) || (a.col() < b.col())); } + else { return ((a.row() == b.row() && a.col() < b.col()) || (a.row() < b.row())); + } } }; } // end namespace internal diff --git a/unsupported/Eigen/src/SparseExtra/MarketIO.h b/unsupported/Eigen/src/SparseExtra/MarketIO.h index 157fbcb1d..a8dabbe8b 100644 --- a/unsupported/Eigen/src/SparseExtra/MarketIO.h +++ b/unsupported/Eigen/src/SparseExtra/MarketIO.h @@ -72,7 +72,8 @@ inline void GetDenseElt(const std::string& line, std::complex& val) template inline void putMarketHeader(std::string& header, int sym) { header = "%%MatrixMarket matrix coordinate "; - if (std::is_same >::value || std::is_same >::value) { + EIGEN_IF_CONSTEXPR( + (std::is_same >::value || std::is_same >::value)) { header += " complex"; if (sym == Symmetric) header += " symmetric"; @@ -80,7 +81,8 @@ inline void putMarketHeader(std::string& header, int sym) { header += " Hermitian"; else header += " general"; - } else { + } + else { header += " real"; if (sym == Symmetric) header += " symmetric"; @@ -338,10 +340,13 @@ bool saveMarketDense(const DenseType& mat, const std::string& filename) { out.flags(std::ios_base::scientific); out.precision(std::numeric_limits::digits10 + 2); - if (std::is_same >::value || std::is_same >::value) + EIGEN_IF_CONSTEXPR( + (std::is_same >::value || std::is_same >::value)) { out << "%%MatrixMarket matrix array complex general\n"; - else + } + else { out << "%%MatrixMarket matrix array real general\n"; + } out << mat.rows() << " " << mat.cols() << "\n"; for (Index i = 0; i < mat.cols(); i++) { for (Index j = 0; j < mat.rows(); j++) { diff --git a/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h b/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h index eb1d6321c..e1d9d542f 100644 --- a/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h +++ b/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h @@ -179,11 +179,12 @@ class MatrixMarketIterator { if (!getMarketHeader(curfile, m_sym, iscomplex, isvector)) continue; if (isvector) continue; if (!iscomplex) { - if (std::is_same >::value || std::is_same >::value) - continue; + EIGEN_IF_CONSTEXPR( + (std::is_same >::value || std::is_same >::value)) + continue; } if (iscomplex) { - if (std::is_same::value || std::is_same::value) continue; + EIGEN_IF_CONSTEXPR((std::is_same::value || std::is_same::value)) { continue; } } // Get the matrix name diff --git a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h index f81193e18..f7029afab 100644 --- a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h +++ b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h @@ -804,20 +804,13 @@ EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar main_igamma_term(Scalar a, S } template -EIGEN_DEVICE_FUNC int igamma_num_iterations() { +EIGEN_DEVICE_FUNC constexpr int igamma_num_iterations() { /* Returns the maximum number of internal iterations for igamma computation. */ - if (mode == VALUE) { - return 2000; - } - - if (std::is_same::value) { - return 200; - } else if (std::is_same::value) { - return 500; - } else { - return 2000; - } + return mode == VALUE ? 2000 + : std::is_same::value ? 200 + : std::is_same::value ? 500 + : 2000; } template @@ -1620,7 +1613,7 @@ struct incbeta_cfe { Scalar ans; int n; - const int num_iters = (std::is_same::value) ? 100 : 300; + constexpr int num_iters = (std::is_same::value) ? 100 : 300; const Scalar thresh = (std::is_same::value) ? machep : Scalar(3) * machep; Scalar r = (std::is_same::value) ? zero : one; diff --git a/unsupported/Eigen/src/Tensor/Tensor.h b/unsupported/Eigen/src/Tensor/Tensor.h index 28e489ac4..134e1b9ea 100644 --- a/unsupported/Eigen/src/Tensor/Tensor.h +++ b/unsupported/Eigen/src/Tensor/Tensor.h @@ -363,9 +363,8 @@ class Tensor : public TensorBase& indices) const { - if (Options & RowMajor) { - return m_storage.dimensions().IndexOfRowMajor(indices); - } else { + EIGEN_IF_CONSTEXPR(Options & RowMajor) { return m_storage.dimensions().IndexOfRowMajor(indices); } + else { return m_storage.dimensions().IndexOfColMajor(indices); } } diff --git a/unsupported/Eigen/src/Tensor/TensorArgMax.h b/unsupported/Eigen/src/Tensor/TensorArgMax.h index 601bdbbd8..8bfe8ac04 100644 --- a/unsupported/Eigen/src/Tensor/TensorArgMax.h +++ b/unsupported/Eigen/src/Tensor/TensorArgMax.h @@ -214,10 +214,11 @@ struct TensorEvaluator, Devic m_impl(op.expression().index_pairs().reduce(op.reduce_dims(), op.reduce_op()), device), m_return_dim(op.return_dim()) { gen_strides(m_orig_impl.dimensions(), m_strides); - if (Layout == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(Layout == static_cast(ColMajor)) { const Index total_size = internal::array_prod(m_orig_impl.dimensions()); m_stride_mod = (m_return_dim < NumDims - 1) ? m_strides[m_return_dim + 1] : total_size; - } else { + } + else { const Index total_size = internal::array_prod(m_orig_impl.dimensions()); m_stride_mod = (m_return_dim > 0) ? m_strides[m_return_dim - 1] : total_size; } @@ -256,12 +257,13 @@ struct TensorEvaluator, Devic // Calculate m_stride_div and m_stride_mod, which are used to // calculate the value of an index w.r.t. the m_return_dim. - if (Layout == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(Layout == static_cast(ColMajor)) { strides[0] = 1; for (int i = 1; i < NumDims; ++i) { strides[i] = strides[i - 1] * dims[i - 1]; } - } else { + } + else { strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { strides[i] = strides[i + 1] * dims[i + 1]; diff --git a/unsupported/Eigen/src/Tensor/TensorAssign.h b/unsupported/Eigen/src/Tensor/TensorAssign.h index 043684368..1e2a3d52d 100644 --- a/unsupported/Eigen/src/Tensor/TensorAssign.h +++ b/unsupported/Eigen/src/Tensor/TensorAssign.h @@ -157,8 +157,8 @@ struct TensorEvaluator, Device> m_leftImpl.coeffRef(i) = m_rightImpl.coeff(i); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalPacket(Index i) const { - const int LhsStoreMode = TensorEvaluator::IsAligned ? Aligned : Unaligned; - const int RhsLoadMode = TensorEvaluator::IsAligned ? Aligned : Unaligned; + constexpr int LhsStoreMode = TensorEvaluator::IsAligned ? Aligned : Unaligned; + constexpr int RhsLoadMode = TensorEvaluator::IsAligned ? Aligned : Unaligned; m_leftImpl.template writePacket(i, m_rightImpl.template packet(i)); } EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const { return m_leftImpl.coeff(index); } diff --git a/unsupported/Eigen/src/Tensor/TensorBlock.h b/unsupported/Eigen/src/Tensor/TensorBlock.h index 66447af93..30b628377 100644 --- a/unsupported/Eigen/src/Tensor/TensorBlock.h +++ b/unsupported/Eigen/src/Tensor/TensorBlock.h @@ -34,12 +34,13 @@ EIGEN_ALWAYS_INLINE DSizes strides(const DSizes(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { strides[0] = 1; for (int i = 1; i < NumDims; ++i) { strides[i] = strides[i - 1] * dimensions[i - 1]; } - } else { + } + else { strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { strides[i] = strides[i + 1] * dimensions[i + 1]; @@ -951,7 +952,7 @@ class StridedLinearBufferCopy { const Scalar* src = &src_data[src_offset]; Scalar* dst = &dst_data[dst_offset]; - if (!Vectorizable) { + EIGEN_IF_CONSTEXPR(!Vectorizable) { for (Index i = 0; i < count; ++i) { dst[i * dst_stride] = src[i * src_stride]; } @@ -961,7 +962,7 @@ class StridedLinearBufferCopy { const IndexType vectorized_size = PacketSize * (count / PacketSize); IndexType i = 0; - if (kind == StridedLinearBufferCopy::Kind::Linear) { + EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Linear) { // ******************************************************************** // // Linear copy from `src` to `dst`. const IndexType unrolled_size = (4 * PacketSize) * (count / (4 * PacketSize)); @@ -976,7 +977,7 @@ class StridedLinearBufferCopy { Packet p = ploadu(src + i); pstoreu(dst + i, p); } - if (HasHalfPacket) { + EIGEN_IF_CONSTEXPR(HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket p = ploadu(src + i); @@ -988,14 +989,15 @@ class StridedLinearBufferCopy { dst[i] = src[i]; } // ******************************************************************** // - } else if (kind == StridedLinearBufferCopy::Kind::Scatter) { + } + else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Scatter) { // Scatter from `src` to `dst`. eigen_assert(src_stride == 1 && dst_stride != 1); for (; i < vectorized_size; i += PacketSize) { Packet p = ploadu(src + i); pscatter(dst + i * dst_stride, p, dst_stride); } - if (HasHalfPacket) { + EIGEN_IF_CONSTEXPR(HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket p = ploadu(src + i); @@ -1007,7 +1009,8 @@ class StridedLinearBufferCopy { dst[i * dst_stride] = src[i]; } // ******************************************************************** // - } else if (kind == StridedLinearBufferCopy::Kind::FillLinear) { + } + else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::FillLinear) { // Fill `dst` with value at `*src`. eigen_assert(src_stride == 0 && dst_stride == 1); @@ -1022,7 +1025,7 @@ class StridedLinearBufferCopy { for (; i < vectorized_size; i += PacketSize) { pstoreu(dst + i, p); } - if (HasHalfPacket) { + EIGEN_IF_CONSTEXPR(HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket hp = pset1(s); @@ -1034,7 +1037,8 @@ class StridedLinearBufferCopy { dst[i] = s; } // ******************************************************************** // - } else if (kind == StridedLinearBufferCopy::Kind::FillScatter) { + } + else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::FillScatter) { // Scatter `*src` into `dst`. eigen_assert(src_stride == 0 && dst_stride != 1); Scalar s = *src; @@ -1042,7 +1046,7 @@ class StridedLinearBufferCopy { for (; i < vectorized_size; i += PacketSize) { pscatter(dst + i * dst_stride, p, dst_stride); } - if (HasHalfPacket) { + EIGEN_IF_CONSTEXPR(HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket hp = pset1(s); @@ -1054,14 +1058,15 @@ class StridedLinearBufferCopy { dst[i * dst_stride] = s; } // ******************************************************************** // - } else if (kind == StridedLinearBufferCopy::Kind::Gather) { + } + else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Gather) { // Gather from `src` into `dst`. eigen_assert(dst_stride == 1); for (; i < vectorized_size; i += PacketSize) { Packet p = pgather(src + i * src_stride, src_stride); pstoreu(dst + i, p); } - if (HasHalfPacket) { + EIGEN_IF_CONSTEXPR(HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket p = pgather(src + i * src_stride, src_stride); @@ -1073,12 +1078,14 @@ class StridedLinearBufferCopy { dst[i] = src[i * src_stride]; } // ******************************************************************** // - } else if (kind == StridedLinearBufferCopy::Kind::Random) { + } + else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Random) { // Random. for (; i < count; ++i) { dst[i * dst_stride] = src[i * src_stride]; } - } else { + } + else { eigen_assert(false); } } @@ -1200,7 +1207,7 @@ class TensorBlockIO { IndexType input_stride = NumDims == 0 ? 1 : src.strides[src_dim_for_dst_stride1_dim]; IndexType output_stride = NumDims == 0 ? 1 : dst.strides[dst_stride1_dim]; - const int at_least_1_dim = NumDims <= 1 ? 1 : NumDims - 1; + constexpr int at_least_1_dim = NumDims <= 1 ? 1 : NumDims - 1; array it; // Initialize block iterator state. Squeeze away any dimension of size 1. @@ -1395,7 +1402,7 @@ class TensorBlockAssignment { // Initialize output inner dimension size based on a layout. const IndexType output_size = NumDims == 0 ? 1 : target.dims.TotalSize(); - const int inner_dim_idx = is_col_major ? 0 : NumDims - 1; + constexpr int inner_dim_idx = is_col_major ? 0 : NumDims - 1; IndexType output_inner_dim_size = target.dims[inner_dim_idx]; // Target inner dimension stride must be '1'. diff --git a/unsupported/Eigen/src/Tensor/TensorBroadcasting.h b/unsupported/Eigen/src/Tensor/TensorBroadcasting.h index 1362db44e..a6fce2998 100644 --- a/unsupported/Eigen/src/Tensor/TensorBroadcasting.h +++ b/unsupported/Eigen/src/Tensor/TensorBroadcasting.h @@ -145,14 +145,15 @@ struct TensorEvaluator, Device> { } } - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_inputStrides[0] = 1; m_outputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1]; m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; } - } else { + } + else { m_inputStrides[NumDims - 1] = 1; m_outputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -217,13 +218,14 @@ struct TensorEvaluator, Device> { return m_impl.coeff(0); } - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { if (isCopy) { return m_impl.coeff(index); } else { return coeffColMajor(index); } - } else { + } + else { if (isCopy) { return m_impl.coeff(index); } else { @@ -312,7 +314,7 @@ struct TensorEvaluator, Device> { return internal::pset1(m_impl.coeff(0)); } - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { if (isCopy) { #ifdef EIGEN_GPU_COMPILE_PHASE // See PR 437: on NVIDIA P100 and K20m we observed a x3-4 speed up by enforcing @@ -330,7 +332,8 @@ struct TensorEvaluator, Device> { } else { return packetColMajor(index); } - } else { + } + else { if (isCopy) { #ifdef EIGEN_GPU_COMPILE_PHASE // See above. @@ -358,10 +361,11 @@ struct TensorEvaluator, Device> { Index startDim, endDim; Index inputIndex, outputOffset, batchedIndex; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { startDim = NumDims - 1; endDim = 1; - } else { + } + else { startDim = 0; endDim = NumDims - 2; } diff --git a/unsupported/Eigen/src/Tensor/TensorChipping.h b/unsupported/Eigen/src/Tensor/TensorChipping.h index 89a9f4a6c..a95de2fd2 100644 --- a/unsupported/Eigen/src/Tensor/TensorChipping.h +++ b/unsupported/Eigen/src/Tensor/TensorChipping.h @@ -156,12 +156,13 @@ struct TensorEvaluator, Device> { m_stride = 1; m_inputStride = 1; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < m_dim.actualDim(); ++i) { m_stride *= input_dims[i]; m_inputStride *= input_dims[i]; } - } else { + } + else { for (int i = NumInputDims - 1; i > m_dim.actualDim(); --i) { m_stride *= input_dims[i]; m_inputStride *= input_dims[i]; @@ -182,10 +183,11 @@ struct TensorEvaluator, Device> { before_chipped_dim_product *= input_dims[i]; } - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_isEffectivelyInnerChipping = before_chipped_dim_product == 1; m_isEffectivelyOuterChipping = after_chipped_dim_product == 1; - } else { + } + else { m_isEffectivelyInnerChipping = after_chipped_dim_product == 1; m_isEffectivelyOuterChipping = before_chipped_dim_product == 1; } diff --git a/unsupported/Eigen/src/Tensor/TensorConcatenation.h b/unsupported/Eigen/src/Tensor/TensorConcatenation.h index 9bff7a24c..cc5146b8a 100644 --- a/unsupported/Eigen/src/Tensor/TensorConcatenation.h +++ b/unsupported/Eigen/src/Tensor/TensorConcatenation.h @@ -164,7 +164,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_leftStrides[0] = 1; m_rightStrides[0] = 1; m_outputStrides[0] = 1; @@ -174,7 +174,8 @@ struct TensorEvaluator subs; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { subs[i] = index / m_outputStrides[i]; index -= subs[i] * m_outputStrides[i]; } subs[0] = index; - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { subs[i] = index / m_outputStrides[i]; index -= subs[i] * m_outputStrides[i]; @@ -377,13 +379,14 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { left_index = subs[0]; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { left_index += (subs[i] % left_dims[i]) * m_leftStrides[i]; } - } else { + } + else { left_index = subs[NumDims - 1]; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { @@ -395,13 +398,14 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { right_index = subs[0]; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { right_index += (subs[i] % right_dims[i]) * m_rightStrides[i]; } - } else { + } + else { right_index = subs[NumDims - 1]; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { @@ -432,7 +436,7 @@ struct TensorEvaluator subs_end; Index remaining = index; Index remaining_end = index + packetSize - 1; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { subs[i] = remaining / m_outputStrides[i]; remaining -= subs[i] * m_outputStrides[i]; @@ -441,7 +445,8 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) ? 0 : NumDims - 1; + constexpr int innermost = (static_cast(Layout) == static_cast(ColMajor)) ? 0 : NumDims - 1; bool packet_in_single_inner_row = true; EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims; ++i) { @@ -471,13 +476,14 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { left_index = subs[0]; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { left_index += subs[i] * m_leftStrides[i]; } - } else { + } + else { left_index = subs[NumDims - 1]; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { @@ -489,13 +495,14 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { right_index = subs[0]; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { right_index += subs[i] * m_rightStrides[i]; } - } else { + } + else { right_index = subs[NumDims - 1]; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { diff --git a/unsupported/Eigen/src/Tensor/TensorContraction.h b/unsupported/Eigen/src/Tensor/TensorContraction.h index e23c48ad2..8aec6e361 100644 --- a/unsupported/Eigen/src/Tensor/TensorContraction.h +++ b/unsupported/Eigen/src/Tensor/TensorContraction.h @@ -447,7 +447,7 @@ struct TensorContractionEvaluatorBase { DSizes eval_left_dims; DSizes eval_right_dims; array, ContractDims> eval_op_indices; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { // For ColMajor, we keep using the existing dimensions for (int i = 0; i < LDims; i++) { eval_left_dims[i] = m_leftImpl.dimensions()[i]; @@ -460,7 +460,8 @@ struct TensorContractionEvaluatorBase { eval_op_indices[i].first = op.indices()[i].first; eval_op_indices[i].second = op.indices()[i].second; } - } else { + } + else { // For RowMajor, we need to reverse the existing dimensions for (int i = 0; i < LDims; i++) { eval_left_dims[i] = m_leftImpl.dimensions()[LDims - i - 1]; @@ -622,7 +623,7 @@ struct TensorContractionEvaluatorBase { } // If the layout is RowMajor, we need to reverse the m_dimensions - if (static_cast(Layout) == static_cast(RowMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(RowMajor)) { for (int i = 0, j = NumDims - 1; i < j; i++, j--) { numext::swap(m_dimensions[i], m_dimensions[j]); } @@ -736,8 +737,8 @@ struct TensorContractionEvaluatorBase { using RightEvaluator = TensorEvaluator; const int lhs_packet_size = internal::unpacket_traits::size; const int rhs_packet_size = internal::unpacket_traits::size; - const int lhs_alignment = LeftEvaluator::IsAligned ? Aligned : Unaligned; - const int rhs_alignment = RightEvaluator::IsAligned ? Aligned : Unaligned; + constexpr int lhs_alignment = LeftEvaluator::IsAligned ? Aligned : Unaligned; + constexpr int rhs_alignment = RightEvaluator::IsAligned ? Aligned : Unaligned; using LhsMapper = internal::TensorContractionInputMapper; diff --git a/unsupported/Eigen/src/Tensor/TensorContractionMapper.h b/unsupported/Eigen/src/Tensor/TensorContractionMapper.h index cebfb0811..d0c3d7a2e 100644 --- a/unsupported/Eigen/src/Tensor/TensorContractionMapper.h +++ b/unsupported/Eigen/src/Tensor/TensorContractionMapper.h @@ -451,7 +451,7 @@ class TensorContractionSubMapper { template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i) const { static_assert(std::is_same::value, "YOU_MADE_A_PROGRAMMING_MISTAKE"); - const int ActualAlignment = (AlignmentType == Aligned) && (Alignment == Aligned) ? Aligned : Unaligned; + constexpr int ActualAlignment = (AlignmentType == Aligned) && (Alignment == Aligned) ? Aligned : Unaligned; if (UseDirectOffsets) { return m_base_mapper.template loadPacket(i, 0); } diff --git a/unsupported/Eigen/src/Tensor/TensorConversion.h b/unsupported/Eigen/src/Tensor/TensorConversion.h index e6cc2e2d2..6b5d04101 100644 --- a/unsupported/Eigen/src/Tensor/TensorConversion.h +++ b/unsupported/Eigen/src/Tensor/TensorConversion.h @@ -374,9 +374,10 @@ struct TensorEvaluator, Device> { // If we are not going to do the cast, we just need to check that base // TensorEvaluator has packet access. Otherwise we also need to make sure, // that we have an implementation of vectorized cast. - const bool Vectorizable = IsSameType ? TensorEvaluator::PacketAccess - : int(TensorEvaluator::PacketAccess) & - int(internal::type_casting_traits::VectorizedCast); + constexpr bool Vectorizable = IsSameType + ? TensorEvaluator::PacketAccess + : int(TensorEvaluator::PacketAccess) & + int(internal::type_casting_traits::VectorizedCast); return internal::PacketConv::run(m_impl, index); diff --git a/unsupported/Eigen/src/Tensor/TensorConvolution.h b/unsupported/Eigen/src/Tensor/TensorConvolution.h index 1d7ef7466..78fc0820f 100644 --- a/unsupported/Eigen/src/Tensor/TensorConvolution.h +++ b/unsupported/Eigen/src/Tensor/TensorConvolution.h @@ -34,14 +34,15 @@ class IndexMapper { array inputStrides; array outputStrides; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { inputStrides[0] = 1; outputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { inputStrides[i] = inputStrides[i - 1] * input_dims[i - 1]; outputStrides[i] = outputStrides[i - 1] * dimensions[i - 1]; } - } else { + } + else { inputStrides[NumDims - 1] = 1; outputStrides[NumDims - 1] = 1; for (int i = static_cast(NumDims) - 2; i >= 0; --i) { @@ -54,7 +55,7 @@ class IndexMapper { array gpuOutputDimensions; array tmp = dimensions; array ordering; - const size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; + constexpr size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; for (int i = 0; i < NumKernelDims; ++i) { const Index index = i + offset; ordering[index] = indices[i]; @@ -78,7 +79,7 @@ class IndexMapper { m_outputStrides[i] = outputStrides[ordering[i]]; } - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumDims; ++i) { if (i > NumKernelDims) { m_gpuInputStrides[i] = m_gpuInputStrides[i - 1] * gpuInputDimensions[i - 1]; @@ -88,7 +89,8 @@ class IndexMapper { m_gpuOutputStrides[i] = 1; } } - } else { + } + else { for (int i = NumDims - 1; i >= 0; --i) { if (i + 1 < static_cast(offset)) { m_gpuInputStrides[i] = m_gpuInputStrides[i + 1] * gpuInputDimensions[i + 1]; @@ -103,7 +105,7 @@ class IndexMapper { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputPlaneToTensorInputOffset(Index p) const { Index inputIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int d = NumDims - 1; d > NumKernelDims; --d) { const Index idx = p / m_gpuInputStrides[d]; inputIndex += idx * m_inputStrides[d]; @@ -112,7 +114,8 @@ class IndexMapper { if (NumKernelDims < NumDims) { inputIndex += p * m_inputStrides[NumKernelDims]; } - } else { + } + else { std::ptrdiff_t limit = 0; if (NumKernelDims < NumDims) { limit = NumDims - NumKernelDims - 1; @@ -129,7 +132,7 @@ class IndexMapper { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputPlaneToTensorOutputOffset(Index p) const { Index outputIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int d = NumDims - 1; d > NumKernelDims; --d) { const Index idx = p / m_gpuOutputStrides[d]; outputIndex += idx * m_outputStrides[d]; @@ -138,7 +141,8 @@ class IndexMapper { if (NumKernelDims < NumDims) { outputIndex += p * m_outputStrides[NumKernelDims]; } - } else { + } + else { std::ptrdiff_t limit = 0; if (NumKernelDims < NumDims) { limit = NumDims - NumKernelDims - 1; @@ -154,32 +158,32 @@ class IndexMapper { } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i) const { - const size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; + constexpr size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_inputStrides[offset]; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i) const { - const size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; + constexpr size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_outputStrides[offset]; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i, Index j) const { - const size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; + constexpr size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_inputStrides[offset] + j * m_inputStrides[offset + 1]; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i, Index j) const { - const size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; + constexpr size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_outputStrides[offset] + j * m_outputStrides[offset + 1]; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i, Index j, Index k) const { - const size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; + constexpr size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_inputStrides[offset] + j * m_inputStrides[offset + 1] + k * m_inputStrides[offset + 2]; } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i, Index j, Index k) const { - const size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; + constexpr size_t offset = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - NumKernelDims; return i * m_outputStrides[offset] + j * m_outputStrides[offset + 1] + k * m_outputStrides[offset + 2]; } @@ -310,12 +314,13 @@ struct TensorEvaluator::Dimensions& input_dims = m_inputImpl.dimensions(); const typename TensorEvaluator::Dimensions& kernel_dims = m_kernelImpl.dimensions(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_inputStride[0] = 1; for (int i = 1; i < NumDims; ++i) { m_inputStride[i] = m_inputStride[i - 1] * input_dims[i - 1]; } - } else { + } + else { m_inputStride[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_inputStride[i] = m_inputStride[i + 1] * input_dims[i + 1]; @@ -323,7 +328,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumKernelDims; ++i) { const Index index = op.indices()[i]; const Index input_dim = input_dims[index]; @@ -342,7 +347,8 @@ struct TensorEvaluator= 0; --i) { const Index index = op.indices()[i]; const Index input_dim = input_dims[index]; @@ -398,7 +404,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / m_outputStride[i]; const Index idx1 = indices[1] / m_outputStride[i]; @@ -407,7 +413,8 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStride[i]; startInput += idx * m_inputStride[i]; index -= idx * m_outputStride[i]; } - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStride[i]; startInput += idx * m_inputStride[i]; @@ -924,8 +932,8 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor) ? 0 : 1; - const int idxY = static_cast(Layout) == static_cast(ColMajor) ? 1 : 0; + constexpr int idxX = static_cast(Layout) == static_cast(ColMajor) ? 0 : 1; + constexpr int idxY = static_cast(Layout) == static_cast(ColMajor) ? 1 : 0; const int kernel_size_x = m_kernelImpl.dimensions()[idxX]; const int kernel_size_y = m_kernelImpl.dimensions()[idxY]; @@ -1013,9 +1021,9 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor) ? 0 : 2; - const int idxY = static_cast(Layout) == static_cast(ColMajor) ? 1 : 1; - const int idxZ = static_cast(Layout) == static_cast(ColMajor) ? 2 : 0; + constexpr int idxX = static_cast(Layout) == static_cast(ColMajor) ? 0 : 2; + constexpr int idxY = static_cast(Layout) == static_cast(ColMajor) ? 1 : 1; + constexpr int idxZ = static_cast(Layout) == static_cast(ColMajor) ? 2 : 0; const int kernel_size_x = m_kernelImpl.dimensions()[idxX]; const int kernel_size_y = m_kernelImpl.dimensions()[idxY]; diff --git a/unsupported/Eigen/src/Tensor/TensorEvaluator.h b/unsupported/Eigen/src/Tensor/TensorEvaluator.h index aef987768..d5fe3396a 100644 --- a/unsupported/Eigen/src/Tensor/TensorEvaluator.h +++ b/unsupported/Eigen/src/Tensor/TensorEvaluator.h @@ -120,18 +120,20 @@ struct TensorEvaluator { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array& coords) const { eigen_assert(m_data != NULL); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { return m_data[m_dims.IndexOfColMajor(coords)]; - } else { + } + else { return m_data[m_dims.IndexOfRowMajor(coords)]; } } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(const array& coords) const { eigen_assert(m_data != NULL); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { return m_data[m_dims.IndexOfColMajor(coords)]; - } else { + } + else { return m_data[m_dims.IndexOfRowMajor(coords)]; } } diff --git a/unsupported/Eigen/src/Tensor/TensorFFT.h b/unsupported/Eigen/src/Tensor/TensorFFT.h index 776564d58..a37fbd7c2 100644 --- a/unsupported/Eigen/src/Tensor/TensorFFT.h +++ b/unsupported/Eigen/src/Tensor/TensorFFT.h @@ -179,12 +179,13 @@ struct TensorEvaluator, D m_dimensions[i] = input_dims[i]; } - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_strides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1]; } - } else { + } + else { m_strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1]; @@ -643,7 +644,7 @@ struct TensorEvaluator, D EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index getBaseOffsetFromIndex(Index index, Index omitted_dim) const { Index result = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > omitted_dim; --i) { const Index partial_m_stride = m_strides[i] / m_dimensions[omitted_dim]; const Index idx = index / partial_m_stride; @@ -651,7 +652,8 @@ struct TensorEvaluator, D result += idx * m_strides[i]; } result += index; - } else { + } + else { for (Index i = 0; i < omitted_dim; ++i) { const Index partial_m_stride = m_strides[i] / m_dimensions[omitted_dim]; const Index idx = index / partial_m_stride; diff --git a/unsupported/Eigen/src/Tensor/TensorFixedSize.h b/unsupported/Eigen/src/Tensor/TensorFixedSize.h index 5006b54d4..696baf297 100644 --- a/unsupported/Eigen/src/Tensor/TensorFixedSize.h +++ b/unsupported/Eigen/src/Tensor/TensorFixedSize.h @@ -203,9 +203,8 @@ class TensorFixedSize : public TensorBase& indices) const { - if (Options & RowMajor) { - return m_storage.dimensions().IndexOfRowMajor(indices); - } else { + EIGEN_IF_CONSTEXPR(Options & RowMajor) { return m_storage.dimensions().IndexOfRowMajor(indices); } + else { return m_storage.dimensions().IndexOfColMajor(indices); } } diff --git a/unsupported/Eigen/src/Tensor/TensorGenerator.h b/unsupported/Eigen/src/Tensor/TensorGenerator.h index ee44dd254..6f7b7b9c5 100644 --- a/unsupported/Eigen/src/Tensor/TensorGenerator.h +++ b/unsupported/Eigen/src/Tensor/TensorGenerator.h @@ -105,14 +105,15 @@ struct TensorEvaluator, Device> { TensorEvaluator argImpl(op.expression(), device); m_dimensions = argImpl.dimensions(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_strides[0] = 1; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1]; if (m_strides[i] != 0) m_fast_strides[i] = IndexDivisor(m_strides[i]); } - } else { + } + else { m_strides[NumDims - 1] = 1; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { @@ -241,14 +242,15 @@ struct TensorEvaluator, Device> { protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void extract_coordinates(Index index, array& coords) const { - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_fast_strides[i]; index -= idx * m_strides[i]; coords[i] = idx; } coords[0] = index; - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_fast_strides[i]; index -= idx * m_strides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorIO.h b/unsupported/Eigen/src/Tensor/TensorIO.h index 381555c37..513864bc3 100644 --- a/unsupported/Eigen/src/Tensor/TensorIO.h +++ b/unsupported/Eigen/src/Tensor/TensorIO.h @@ -238,9 +238,8 @@ struct TensorPrinter { if (fmt.precision == StreamPrecision) { explicit_precision = 0; } else if (fmt.precision == FullPrecision) { - if (NumTraits::IsInteger) { - explicit_precision = 0; - } else { + EIGEN_IF_CONSTEXPR(NumTraits::IsInteger) { explicit_precision = 0; } + else { explicit_precision = significant_decimals_impl::run(); } } else { @@ -386,9 +385,8 @@ struct TensorPrinter { if (fmt.precision == StreamPrecision) { explicit_precision = 0; } else if (fmt.precision == FullPrecision) { - if (NumTraits::IsInteger) { - explicit_precision = 0; - } else { + EIGEN_IF_CONSTEXPR(NumTraits::IsInteger) { explicit_precision = 0; } + else { explicit_precision = significant_decimals_impl::run(); } } else { diff --git a/unsupported/Eigen/src/Tensor/TensorImagePatch.h b/unsupported/Eigen/src/Tensor/TensorImagePatch.h index 7ae4a7997..ddcc9335d 100644 --- a/unsupported/Eigen/src/Tensor/TensorImagePatch.h +++ b/unsupported/Eigen/src/Tensor/TensorImagePatch.h @@ -193,11 +193,12 @@ struct TensorEvaluator, Device> { const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); // Caches a few variables. - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_inputDepth = input_dims[0]; m_inputRows = input_dims[1]; m_inputCols = input_dims[2]; - } else { + } + else { m_inputDepth = input_dims[NumInputDims - 1]; m_inputRows = input_dims[NumInputDims - 2]; m_inputCols = input_dims[NumInputDims - 3]; @@ -269,7 +270,7 @@ struct TensorEvaluator, Device> { eigen_assert(m_outputCols > 0); // Dimensions for result of extraction. - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { // ColMajor // 0: depth // 1: patch_rows @@ -283,7 +284,8 @@ struct TensorEvaluator, Device> { for (int i = 4; i < NumDims; ++i) { m_dimensions[i] = input_dims[i - 1]; } - } else { + } + else { // RowMajor // NumDims-1: depth // NumDims-2: patch_rows @@ -300,11 +302,12 @@ struct TensorEvaluator, Device> { } // Strides for moving the patch in various dimensions. - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_colStride = m_dimensions[1]; m_patchStride = m_colStride * m_dimensions[2] * m_dimensions[0]; m_otherStride = m_patchStride * m_dimensions[3]; - } else { + } + else { m_colStride = m_dimensions[NumDims - 2]; m_patchStride = m_colStride * m_dimensions[NumDims - 3] * m_dimensions[NumDims - 1]; m_otherStride = m_patchStride * m_dimensions[NumDims - 4]; @@ -325,9 +328,10 @@ struct TensorEvaluator, Device> { // Number of patches in the width dimension. m_fastOutputRows = internal::TensorIntDivisor(m_outputRows); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_fastOutputDepth = internal::TensorIntDivisor(m_dimensions[0]); - } else { + } + else { m_fastOutputDepth = internal::TensorIntDivisor(m_dimensions[NumDims - 1]); } } @@ -361,7 +365,7 @@ struct TensorEvaluator, Device> { // Compute the remainder within the patch once, then derive both // patchOffset and depth from it without an extra division. - const int depth_index = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - 1; + constexpr int depth_index = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - 1; const Index patchRemainder = index - otherIndex * m_otherStride - patch2DIndex * m_patchStride; const Index patchOffset = patchRemainder / m_fastOutputDepth; const Index depth = patchRemainder - patchOffset * m_dimensions[depth_index]; @@ -397,7 +401,7 @@ struct TensorEvaluator, Device> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { eigen_assert(index + PacketSize - 1 < dimensions().TotalSize()); - const int depth_index = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - 1; + constexpr int depth_index = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - 1; const Index lastIdx = index + PacketSize - 1; // Decompose index into (otherIndex, patch2DIndex, patchRemainder). diff --git a/unsupported/Eigen/src/Tensor/TensorInflation.h b/unsupported/Eigen/src/Tensor/TensorInflation.h index 088cab1b0..2550d12ac 100644 --- a/unsupported/Eigen/src/Tensor/TensorInflation.h +++ b/unsupported/Eigen/src/Tensor/TensorInflation.h @@ -111,14 +111,15 @@ struct TensorEvaluator, Device> { } const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_outputStrides[0] = 1; m_inputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1]; } - } else { // RowMajor + } + else { // RowMajor m_outputStrides[NumDims - 1] = 1; m_inputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -141,7 +142,7 @@ struct TensorEvaluator, Device> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool getInputIndex(Index index, Index* inputIndex) const { eigen_assert(index < dimensions().TotalSize()); *inputIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; @@ -156,7 +157,8 @@ struct TensorEvaluator, Device> { } *inputIndex += index / m_strides[0]; return true; - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorMap.h b/unsupported/Eigen/src/Tensor/TensorMap.h index 639e4c18b..eec3744dc 100644 --- a/unsupported/Eigen/src/Tensor/TensorMap.h +++ b/unsupported/Eigen/src/Tensor/TensorMap.h @@ -105,10 +105,11 @@ class TensorMap : public TensorBase& indices) const { - if (PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(indices); return m_data[index]; - } else { + } + else { const Index index = m_dimensions.IndexOfColMajor(indices); return m_data[index]; } @@ -129,11 +130,12 @@ class TensorMap : public TensorBase::highest() >= otherIndices)...)); - if (PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(array{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; - } else { + } + else { const Index index = m_dimensions.IndexOfColMajor(array{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; @@ -141,10 +143,11 @@ class TensorMap : public TensorBase& indices) { - if (PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(indices); return m_data[index]; - } else { + } + else { const Index index = m_dimensions.IndexOfColMajor(indices); return m_data[index]; } @@ -167,11 +170,12 @@ class TensorMap : public TensorBase::highest() >= otherIndices)...)); const std::size_t NumDims = sizeof...(otherIndices) + 2; - if (PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(array{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; - } else { + } + else { const Index index = m_dimensions.IndexOfColMajor(array{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; diff --git a/unsupported/Eigen/src/Tensor/TensorMorphing.h b/unsupported/Eigen/src/Tensor/TensorMorphing.h index 5734bc85f..2be1637ae 100644 --- a/unsupported/Eigen/src/Tensor/TensorMorphing.h +++ b/unsupported/Eigen/src/Tensor/TensorMorphing.h @@ -399,7 +399,7 @@ struct TensorEvaluator, Devi const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); const Sizes& output_dims = op.sizes(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_inputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1]; @@ -411,7 +411,8 @@ struct TensorEvaluator, Devi m_outputStrides[i] = m_outputStrides[i - 1] * output_dims[i - 1]; m_fastOutputStrides[i] = internal::TensorIntDivisor(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1); } - } else { + } + else { m_inputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_inputStrides[i] = m_inputStrides[i + 1] * input_dims[i + 1]; @@ -432,14 +433,15 @@ struct TensorEvaluator, Devi m_impl.evalSubExprsIfNeeded(NULL); if (!NumTraits>::RequireInitialization && data && m_impl.data()) { Index contiguous_values = 1; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumDims; ++i) { contiguous_values *= dimensions()[i]; if (dimensions()[i] != m_impl.dimensions()[i]) { break; } } - } else { + } + else { for (int i = NumDims - 1; i >= 0; --i) { contiguous_values *= dimensions()[i]; if (dimensions()[i] != m_impl.dimensions()[i]) { @@ -491,7 +493,7 @@ struct TensorEvaluator, Devi Index inputIndices[] = {0, 0}; Index indices[] = {index, index + packetSize - 1}; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / m_fastOutputStrides[i]; @@ -503,7 +505,8 @@ struct TensorEvaluator, Devi } inputIndices[0] += (indices[0] + m_offsets[0]); inputIndices[1] += (indices[1] + m_offsets[0]); - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx0 = indices[0] / m_fastOutputStrides[i]; @@ -554,7 +557,7 @@ struct TensorEvaluator, Devi typename Storage::Type result = constCast(m_impl.data()); if (result) { Index offset = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumDims; ++i) { if (m_dimensions[i] != m_impl.dimensions()[i]) { offset += m_offsets[i] * m_inputStrides[i]; @@ -567,7 +570,8 @@ struct TensorEvaluator, Devi break; } } - } else { + } + else { for (int i = NumDims - 1; i >= 0; --i) { if (m_dimensions[i] != m_impl.dimensions()[i]) { offset += m_offsets[i] * m_inputStrides[i]; @@ -589,7 +593,7 @@ struct TensorEvaluator, Devi protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const { Index inputIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_fastOutputStrides[i]; @@ -597,7 +601,8 @@ struct TensorEvaluator, Devi index -= idx * m_outputStrides[i]; } inputIndex += (index + m_offsets[0]); - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_fastOutputStrides[i]; @@ -670,7 +675,7 @@ struct TensorEvaluator, Device> const int packetSize = PacketType::size; Index inputIndices[] = {0, 0}; Index indices[] = {index, index + packetSize - 1}; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / this->m_fastOutputStrides[i]; @@ -682,7 +687,8 @@ struct TensorEvaluator, Device> } inputIndices[0] += (indices[0] + this->m_offsets[0]); inputIndices[1] += (indices[1] + this->m_offsets[0]); - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx0 = indices[0] / this->m_fastOutputStrides[i]; @@ -838,7 +844,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_inputStrides[0] = m_strides[0]; m_offsets[0] = startIndicesClamped[0]; Index previousDimProduct = 1; @@ -854,7 +860,8 @@ struct TensorEvaluator(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1); } - } else { + } + else { m_inputStrides[NumDims - 1] = m_strides[NumDims - 1]; m_offsets[NumDims - 1] = startIndicesClamped[NumDims - 1]; Index previousDimProduct = 1; @@ -898,14 +905,15 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i >= 0; --i) { const Index idx = index / m_fastOutputStrides[i]; inputIndex += idx * m_inputStrides[i] + m_offsets[i]; index -= idx * m_outputStrides[i]; } - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims; ++i) { const Index idx = index / m_fastOutputStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorPadding.h b/unsupported/Eigen/src/Tensor/TensorPadding.h index dec3d53bb..fda6932b9 100644 --- a/unsupported/Eigen/src/Tensor/TensorPadding.h +++ b/unsupported/Eigen/src/Tensor/TensorPadding.h @@ -121,7 +121,7 @@ struct TensorEvaluator, Device m_dimensions[i] += m_padding[i].first + m_padding[i].second; } const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_inputStrides[0] = 1; m_outputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { @@ -129,7 +129,8 @@ struct TensorEvaluator, Device m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; } m_outputStrides[NumDims] = m_outputStrides[NumDims - 1] * m_dimensions[NumDims - 1]; - } else { + } + else { m_inputStrides[NumDims - 1] = 1; m_outputStrides[NumDims] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -159,7 +160,7 @@ struct TensorEvaluator, Device EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { eigen_assert(index < dimensions().TotalSize()); Index inputIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; @@ -173,7 +174,8 @@ struct TensorEvaluator, Device return m_paddingValue; } inputIndex += (index - m_padding[0].first); - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStrides[i + 1]; @@ -193,18 +195,17 @@ struct TensorEvaluator, Device template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - if (static_cast(Layout) == static_cast(ColMajor)) { - return packetColMajor(index); - } + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { return packetColMajor(index); } return packetRowMajor(index); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { TensorOpCost cost = m_impl.costPerCoeff(vectorized); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims; ++i) updateCostPerDimension(cost, i, i == 0); - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i >= 0; --i) updateCostPerDimension(cost, i, i == NumDims - 1); } @@ -225,7 +226,7 @@ struct TensorEvaluator, Device } static constexpr bool IsColMajor = Layout == static_cast(ColMajor); - const int inner_dim_idx = IsColMajor ? 0 : NumDims - 1; + constexpr int inner_dim_idx = IsColMajor ? 0 : NumDims - 1; Index offset = desc.offset(); @@ -342,7 +343,7 @@ struct TensorEvaluator, Device // and equal to the block inner dimension (input_inner_dim_size == output_inner_dim_size); - const int squeeze_dim = IsColMajor ? inner_dim_idx + 1 : inner_dim_idx - 1; + constexpr int squeeze_dim = IsColMajor ? inner_dim_idx + 1 : inner_dim_idx - 1; // Maximum coordinate on a squeeze dimension that we can write to. const Index squeeze_max_coord = diff --git a/unsupported/Eigen/src/Tensor/TensorPatch.h b/unsupported/Eigen/src/Tensor/TensorPatch.h index d8eaa35fe..1826d9bd0 100644 --- a/unsupported/Eigen/src/Tensor/TensorPatch.h +++ b/unsupported/Eigen/src/Tensor/TensorPatch.h @@ -101,7 +101,7 @@ struct TensorEvaluator, Device> { Index num_patches = 1; const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); const PatchDim& patch_dims = op.patch_dims(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumDims - 1; ++i) { m_dimensions[i] = patch_dims[i]; num_patches *= (input_dims[i] - patch_dims[i] + 1); @@ -118,7 +118,8 @@ struct TensorEvaluator, Device> { for (int i = 1; i < NumDims; ++i) { m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; } - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { m_dimensions[i + 1] = patch_dims[i]; num_patches *= (input_dims[i] - patch_dims[i] + 1); @@ -154,7 +155,7 @@ struct TensorEvaluator, Device> { // Find the offset of the element wrt the location of the first element. Index patchOffset = index - patchIndex * m_outputStrides[output_stride_index]; Index inputIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i > 0; --i) { const Index patchIdx = patchIndex / m_patchStrides[i]; @@ -163,7 +164,8 @@ struct TensorEvaluator, Device> { patchOffset -= offsetIdx * m_outputStrides[i]; inputIndex += (patchIdx + offsetIdx) * m_inputStrides[i]; } - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 2; ++i) { const Index patchIdx = patchIndex / m_patchStrides[i]; @@ -189,7 +191,7 @@ struct TensorEvaluator, Device> { indices[1] - patchIndices[1] * m_outputStrides[output_stride_index]}; Index inputIndices[2] = {0, 0}; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i > 0; --i) { const Index patchIdx[2] = {patchIndices[0] / m_patchStrides[i], patchIndices[1] / m_patchStrides[i]}; @@ -203,7 +205,8 @@ struct TensorEvaluator, Device> { inputIndices[0] += (patchIdx[0] + offsetIdx[0]) * m_inputStrides[i]; inputIndices[1] += (patchIdx[1] + offsetIdx[1]) * m_inputStrides[i]; } - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 2; ++i) { const Index patchIdx[2] = {patchIndices[0] / m_patchStrides[i], patchIndices[1] / m_patchStrides[i]}; diff --git a/unsupported/Eigen/src/Tensor/TensorReduction.h b/unsupported/Eigen/src/Tensor/TensorReduction.h index 8f677f1a3..1ac2c4b16 100644 --- a/unsupported/Eigen/src/Tensor/TensorReduction.h +++ b/unsupported/Eigen/src/Tensor/TensorReduction.h @@ -607,13 +607,14 @@ struct TensorReductionEvaluatorBase 0) { - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_outputStrides[0] = 1; for (int i = 1; i < NumOutputDims; ++i) { m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; m_fastOutputStrides[i] = internal::TensorIntDivisor(m_outputStrides[i]); } - } else { + } + else { m_outputStrides[static_cast(NumOutputDims - 1)] = 1; for (int i = NumOutputDims - 2; i >= 0; --i) { m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; @@ -625,12 +626,13 @@ struct TensorReductionEvaluatorBase 0) { array input_strides; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { input_strides[0] = 1; for (int i = 1; i < NumInputDims; ++i) { input_strides[i] = input_strides[i - 1] * input_dims[i - 1]; } - } else { + } + else { input_strides.back() = 1; for (int i = NumInputDims - 2; i >= 0; --i) { input_strides[i] = input_strides[i + 1] * input_dims[i + 1]; @@ -693,9 +695,10 @@ struct TensorReductionEvaluatorBase= 3)) || (RunningOnSycl)) { bool reducing_inner_dims = true; for (int i = 0; i < NumReducedDims; ++i) { - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { reducing_inner_dims &= m_reduced[i]; - } else { + } + else { reducing_inner_dims &= m_reduced[NumInputDims - 1 - i]; } } @@ -730,9 +733,10 @@ struct TensorReductionEvaluatorBase(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { preserving_inner_dims &= m_reduced[NumInputDims - 1 - i]; - } else { + } + else { preserving_inner_dims &= m_reduced[i]; } } @@ -846,7 +850,7 @@ struct TensorReductionEvaluatorBase(Layout) == static_cast(ColMajor)) ? 0 : NumOutputDims - 1; + constexpr int innermost_dim = (static_cast(Layout) == static_cast(ColMajor)) ? 0 : NumOutputDims - 1; // TBD: extend this to the n innermost dimensions that we preserve. if (((firstIndex % m_dimensions[innermost_dim]) + PacketSize - 1) < m_dimensions[innermost_dim]) { Op reducer(m_reducer); @@ -942,15 +946,16 @@ struct TensorReductionEvaluatorBase(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { return index * m_preservedStrides[0]; - } else { + } + else { return index * m_preservedStrides[NumPreservedStrides - 1]; } } // TBD: optimize the case where we preserve the innermost dimensions. Index startInput = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumOutputDims - 1; i > 0; --i) { // This is index_i in the output tensor. const Index idx = index / m_outputStrides[i]; @@ -963,7 +968,8 @@ struct TensorReductionEvaluatorBase __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) { - if (sizeof(T) == 4) { + EIGEN_IF_CONSTEXPR(sizeof(T) == 4) { unsigned int oldval = *reinterpret_cast(output); unsigned int newval = oldval; reducer.reduce(accum, reinterpret_cast(&newval)); @@ -42,7 +42,8 @@ __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) return; } } - } else if (sizeof(T) == 8) { + } + else EIGEN_IF_CONSTEXPR(sizeof(T) == 8) { unsigned long long oldval = *reinterpret_cast(output); unsigned long long newval = oldval; reducer.reduce(accum, reinterpret_cast(&newval)); @@ -58,7 +59,8 @@ __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) return; } } - } else { + } + else { gpu_assert(0 && "Wordsize not supported"); } } @@ -365,9 +367,8 @@ template __global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void ReductionCleanupKernelHalfFloat(Op reducer, half* output, half* scratch) { eigen_assert(threadIdx.x == 1); typedef packet_traits::type packet_type; - if (unpacket_traits::size == 1) { - *output = *scratch; - } else { + EIGEN_IF_CONSTEXPR(unpacket_traits::size == 1) { *output = *scratch; } + else { half2* pscratch = reinterpret_cast(scratch); half tmp = __float2half(0.f); for (int i = 0; i < unpacket_traits::size; i += 2) { diff --git a/unsupported/Eigen/src/Tensor/TensorRef.h b/unsupported/Eigen/src/Tensor/TensorRef.h index 3a45bb520..6ca1b4092 100644 --- a/unsupported/Eigen/src/Tensor/TensorRef.h +++ b/unsupported/Eigen/src/Tensor/TensorRef.h @@ -194,12 +194,13 @@ class TensorRefBase : public TensorBase { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(const array& indices) const { const Dimensions& dims = this->dimensions(); Index index = 0; - if (PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { index += indices[0]; for (size_t i = 1; i < NumIndices; ++i) { index = index * dims[i] + indices[i]; } - } else { + } + else { index += indices[NumIndices - 1]; for (int i = NumIndices - 2; i >= 0; --i) { index = index * dims[i] + indices[i]; @@ -277,12 +278,13 @@ class TensorRef : public internal::TensorRefBase> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(const array& indices) { const Dimensions& dims = this->dimensions(); Index index = 0; - if (PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { index += indices[0]; for (size_t i = 1; i < NumIndices; ++i) { index = index * dims[i] + indices[i]; } - } else { + } + else { index += indices[NumIndices - 1]; for (int i = NumIndices - 2; i >= 0; --i) { index = index * dims[i] + indices[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorReverse.h b/unsupported/Eigen/src/Tensor/TensorReverse.h index ac6fe0724..dfa566b1d 100644 --- a/unsupported/Eigen/src/Tensor/TensorReverse.h +++ b/unsupported/Eigen/src/Tensor/TensorReverse.h @@ -116,13 +116,14 @@ struct TensorEvaluator, Device // Compute strides m_dimensions = m_impl.dimensions(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_strides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1]; if (m_strides[i] > 0) m_fastStrides[i] = IndexDivisor(m_strides[i]); } - } else { + } + else { m_strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1]; @@ -150,7 +151,7 @@ struct TensorEvaluator, Device EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index reverseIndex(Index index) const { eigen_assert(index < dimensions().TotalSize()); Index inputIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { Index idx = index / m_fastStrides[i]; @@ -165,7 +166,8 @@ struct TensorEvaluator, Device } else { inputIndex += index; } - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { Index idx = index / m_fastStrides[i]; @@ -230,7 +232,7 @@ struct TensorEvaluator, Device // access into the underlying tensor expression. static const bool isColMajor = static_cast(Layout) == static_cast(ColMajor); - static const Index inner_dim_idx = isColMajor ? 0 : NumDims - 1; + static constexpr Index inner_dim_idx = isColMajor ? 0 : NumDims - 1; const bool inner_dim_reversed = m_reverse[inner_dim_idx]; // Offset in the output block. diff --git a/unsupported/Eigen/src/Tensor/TensorRoll.h b/unsupported/Eigen/src/Tensor/TensorRoll.h index fd33451a6..822c2fc7d 100644 --- a/unsupported/Eigen/src/Tensor/TensorRoll.h +++ b/unsupported/Eigen/src/Tensor/TensorRoll.h @@ -111,13 +111,14 @@ struct TensorEvaluator, Device> { // Compute strides m_dimensions = m_impl.dimensions(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_strides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1]; if (m_strides[i] > 0) m_fast_strides[i] = IndexDivisor(m_strides[i]); } - } else { + } + else { m_strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1]; @@ -163,7 +164,7 @@ struct TensorEvaluator, Device> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rollIndex(Index index) const { eigen_assert(index < dimensions().TotalSize()); Index rolledIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { Index idx = index / m_fast_strides[i]; @@ -171,7 +172,8 @@ struct TensorEvaluator, Device> { rolledIndex += roll(idx, m_rolls[i], m_dimensions[i]) * m_strides[i]; } rolledIndex += roll(index, m_rolls[0], m_dimensions[0]); - } else { + } + else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { Index idx = index / m_fast_strides[i]; @@ -252,7 +254,7 @@ struct TensorEvaluator, Device> { const typename TensorBlock::Storage block_storage = TensorBlock::prepareStorage(desc, scratch); CoeffReturnType* block_buffer = block_storage.data(); - static const int inner_dim = is_col_major ? 0 : NumDims - 1; + static constexpr int inner_dim = is_col_major ? 0 : NumDims - 1; const Index inner_dim_size = it[0].size; while (it[NumDims - 1].count < it[NumDims - 1].size) { @@ -305,14 +307,15 @@ struct TensorEvaluator, Device> { const Device EIGEN_DEVICE_REF m_device; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void extract_coordinates(Index index, array& coords) const { - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_fast_strides[i]; index -= idx * m_strides[i]; coords[i] = idx; } coords[0] = index; - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_fast_strides[i]; index -= idx * m_strides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorScan.h b/unsupported/Eigen/src/Tensor/TensorScan.h index a6e72533c..75350aeec 100644 --- a/unsupported/Eigen/src/Tensor/TensorScan.h +++ b/unsupported/Eigen/src/Tensor/TensorScan.h @@ -398,11 +398,12 @@ struct TensorEvaluator, Device> { // Compute stride of scan axis const Dimensions& dims = m_impl.dimensions(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < op.axis(); ++i) { m_stride = m_stride * dims[i]; } - } else { + } + else { // dims can only be indexed through unsigned integers, // so use an unsigned type to let the compiler know. // This prevents spurious warnings: "'*((void*)(& evaluator)+64)[18446744073709551615]' may be used uninitialized diff --git a/unsupported/Eigen/src/Tensor/TensorShuffling.h b/unsupported/Eigen/src/Tensor/TensorShuffling.h index 6db7a3cbe..40696ff32 100644 --- a/unsupported/Eigen/src/Tensor/TensorShuffling.h +++ b/unsupported/Eigen/src/Tensor/TensorShuffling.h @@ -120,7 +120,7 @@ struct TensorEvaluator, Device> { } } - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_unshuffledInputStrides[0] = 1; m_outputStrides[0] = 1; @@ -130,7 +130,8 @@ struct TensorEvaluator, Device> { m_fastOutputStrides[i] = internal::TensorIntDivisor(m_outputStrides[i] > 0 ? m_outputStrides[i] : Index(1)); } - } else { + } + else { m_unshuffledInputStrides[NumDims - 1] = 1; m_outputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -264,14 +265,15 @@ struct TensorEvaluator, Device> { const DSizes& output_block_strides, const DSizes, NumDims>& fast_input_block_strides) const { Index output_index = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = input_index / fast_input_block_strides[i]; output_index += idx * output_block_strides[m_inverseShuffle[i]]; input_index -= idx * input_block_strides[i]; } return output_index + input_index * output_block_strides[m_inverseShuffle[0]]; - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = input_index / fast_input_block_strides[i]; output_index += idx * output_block_strides[m_inverseShuffle[i]]; @@ -283,14 +285,15 @@ struct TensorEvaluator, Device> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const { Index inputIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_fastOutputStrides[i]; inputIndex += idx * m_inputStrides[i]; index -= idx * m_outputStrides[i]; } return inputIndex + index * m_inputStrides[0]; - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_fastOutputStrides[i]; inputIndex += idx * m_inputStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorStriding.h b/unsupported/Eigen/src/Tensor/TensorStriding.h index 9292d7442..fa82e1eaf 100644 --- a/unsupported/Eigen/src/Tensor/TensorStriding.h +++ b/unsupported/Eigen/src/Tensor/TensorStriding.h @@ -107,7 +107,7 @@ struct TensorEvaluator, Device> { } const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_outputStrides[0] = 1; m_inputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { @@ -116,7 +116,8 @@ struct TensorEvaluator, Device> { m_inputStrides[i - 1] *= op.strides()[i - 1]; } m_inputStrides[NumDims - 1] *= op.strides()[NumDims - 1]; - } else { // RowMajor + } + else { // RowMajor m_outputStrides[NumDims - 1] = 1; m_inputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -147,7 +148,7 @@ struct TensorEvaluator, Device> { Index inputIndices[] = {0, 0}; Index indices[] = {index, index + PacketSize - 1}; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / m_outputStrides[i]; @@ -159,7 +160,8 @@ struct TensorEvaluator, Device> { } inputIndices[0] += indices[0] * m_inputStrides[0]; inputIndices[1] += indices[1] * m_inputStrides[0]; - } else { // RowMajor + } + else { // RowMajor EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx0 = indices[0] / m_outputStrides[i]; @@ -195,7 +197,7 @@ struct TensorEvaluator, Device> { if (vectorized) { compute_cost *= 2; // packet() computes two indices } - const int innerDim = (static_cast(Layout) == static_cast(ColMajor)) ? 0 : (NumDims - 1); + constexpr int innerDim = (static_cast(Layout) == static_cast(ColMajor)) ? 0 : (NumDims - 1); return m_impl.costPerCoeff(vectorized && m_inputStrides[innerDim] == 1) + // Computation is not vectorized per se, but it is done once per packet. TensorOpCost(0, 0, compute_cost, vectorized, PacketSize); @@ -206,7 +208,7 @@ struct TensorEvaluator, Device> { protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const { Index inputIndex = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; @@ -214,7 +216,8 @@ struct TensorEvaluator, Device> { index -= idx * m_outputStrides[i]; } inputIndex += index * m_inputStrides[0]; - } else { // RowMajor + } + else { // RowMajor EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStrides[i]; @@ -268,7 +271,7 @@ struct TensorEvaluator, Device> Index inputIndices[] = {0, 0}; Index indices[] = {index, index + PacketSize - 1}; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / this->m_outputStrides[i]; @@ -280,7 +283,8 @@ struct TensorEvaluator, Device> } inputIndices[0] += indices[0] * this->m_inputStrides[0]; inputIndices[1] += indices[1] * this->m_inputStrides[0]; - } else { // RowMajor + } + else { // RowMajor EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx0 = indices[0] / this->m_outputStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorTrace.h b/unsupported/Eigen/src/Tensor/TensorTrace.h index 46d24b0e7..f8953d682 100644 --- a/unsupported/Eigen/src/Tensor/TensorTrace.h +++ b/unsupported/Eigen/src/Tensor/TensorTrace.h @@ -159,12 +159,13 @@ struct TensorEvaluator, Device> { // Compute the output strides if (NumOutputDims > 0) { - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_outputStrides[0] = 1; for (int i = 1; i < NumOutputDims; ++i) { m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; } - } else { + } + else { m_outputStrides.back() = 1; for (int i = NumOutputDims - 2; i >= 0; --i) { m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; @@ -175,12 +176,13 @@ struct TensorEvaluator, Device> { // Compute the input strides if (NumInputDims > 0) { array input_strides; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { input_strides[0] = 1; for (int i = 1; i < NumInputDims; ++i) { input_strides[i] = input_strides[i - 1] * input_dims[i - 1]; } - } else { + } + else { input_strides.back() = 1; for (int i = NumInputDims - 2; i >= 0; --i) { input_strides[i] = input_strides[i + 1] * input_dims[i + 1]; @@ -247,14 +249,15 @@ struct TensorEvaluator, Device> { // Given the output index, finds the first index in the input tensor used to compute the trace EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const { Index startInput = 0; - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumOutputDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; startInput += idx * m_preservedStrides[i]; index -= idx * m_outputStrides[i]; } startInput += index * m_preservedStrides[0]; - } else { + } + else { for (int i = 0; i < NumOutputDims - 1; ++i) { const Index idx = index / m_outputStrides[i]; startInput += idx * m_preservedStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorVolumePatch.h b/unsupported/Eigen/src/Tensor/TensorVolumePatch.h index 411adb9db..e55cac38e 100644 --- a/unsupported/Eigen/src/Tensor/TensorVolumePatch.h +++ b/unsupported/Eigen/src/Tensor/TensorVolumePatch.h @@ -209,12 +209,13 @@ struct TensorEvaluator, D const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); // Cache a few variables. - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_inputDepth = input_dims[0]; m_inputPlanes = input_dims[1]; m_inputRows = input_dims[2]; m_inputCols = input_dims[3]; - } else { + } + else { m_inputDepth = input_dims[NumInputDims - 1]; m_inputPlanes = input_dims[NumInputDims - 2]; m_inputRows = input_dims[NumInputDims - 3]; @@ -287,7 +288,7 @@ struct TensorEvaluator, D eigen_assert(m_outputPlanes > 0); // Dimensions for result of extraction. - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { // ColMajor // 0: depth // 1: patch_planes @@ -303,7 +304,8 @@ struct TensorEvaluator, D for (int i = 5; i < NumDims; ++i) { m_dimensions[i] = input_dims[i - 1]; } - } else { + } + else { // RowMajor // NumDims-1: depth // NumDims-2: patch_planes @@ -322,12 +324,13 @@ struct TensorEvaluator, D } // Strides for the output tensor. - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_rowStride = m_dimensions[1]; m_colStride = m_dimensions[2] * m_rowStride; m_patchStride = m_colStride * m_dimensions[3] * m_dimensions[0]; m_otherStride = m_patchStride * m_dimensions[4]; - } else { + } + else { m_rowStride = m_dimensions[NumDims - 2]; m_colStride = m_dimensions[NumDims - 3] * m_rowStride; m_patchStride = m_colStride * m_dimensions[NumDims - 4] * m_dimensions[NumDims - 1]; @@ -355,9 +358,10 @@ struct TensorEvaluator, D m_fastOutputPlanes = internal::TensorIntDivisor(m_outputPlanes); m_fastOutputPlanesRows = internal::TensorIntDivisor(m_outputPlanesRows); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { m_fastOutputDepth = internal::TensorIntDivisor(m_dimensions[0]); - } else { + } + else { m_fastOutputDepth = internal::TensorIntDivisor(m_dimensions[NumDims - 1]); } } @@ -423,7 +427,7 @@ struct TensorEvaluator, D return Scalar(m_paddingValue); } - const int depth_index = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - 1; + constexpr int depth_index = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - 1; const Index depth = index - (index / m_fastOutputDepth) * m_dimensions[depth_index]; const Index inputIndex = depth + origInputRow * m_rowInputStride + origInputCol * m_colInputStride + @@ -502,7 +506,7 @@ struct TensorEvaluator, D if (inputPlanes[0] >= 0 && inputPlanes[1] < m_inputPlanes) { // no padding - const int depth_index = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - 1; + constexpr int depth_index = static_cast(Layout) == static_cast(ColMajor) ? 0 : NumDims - 1; const Index depth = index - (index / m_fastOutputDepth) * m_dimensions[depth_index]; const Index inputIndex = depth + inputRows[0] * m_rowInputStride + inputCols[0] * m_colInputStride + m_planeInputStride * inputPlanes[0] + otherIndex * m_otherInputStride; diff --git a/unsupported/test/sparse_extra.cpp b/unsupported/test/sparse_extra.cpp index 7a2ac9a0b..93c893771 100644 --- a/unsupported/test/sparse_extra.cpp +++ b/unsupported/test/sparse_extra.cpp @@ -75,8 +75,8 @@ void sparse_extra(const SparseMatrixType& ref) { // test coeff and coeffRef for (int i = 0; i < (int)zeroCoords.size(); ++i) { VERIFY_IS_MUCH_SMALLER_THAN(m.coeff(zeroCoords[i].x(), zeroCoords[i].y()), eps); - if (std::is_same >::value) - VERIFY_RAISES_ASSERT(m.coeffRef(zeroCoords[0].x(), zeroCoords[0].y()) = 5); + EIGEN_IF_CONSTEXPR((std::is_same >::value)) + VERIFY_RAISES_ASSERT(m.coeffRef(zeroCoords[0].x(), zeroCoords[0].y()) = 5); } VERIFY_IS_APPROX(m, refMat); diff --git a/unsupported/test/tensor_block_access.cpp b/unsupported/test/tensor_block_access.cpp index 635ce6ab2..ed01cd161 100644 --- a/unsupported/test/tensor_block_access.cpp +++ b/unsupported/test/tensor_block_access.cpp @@ -144,14 +144,15 @@ template static Index GetInputIndex(Index output_index, const array& output_to_input_dim_map, const array& input_strides, const array& output_strides) { int input_index = 0; - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = output_index / output_strides[i]; input_index += idx * input_strides[output_to_input_dim_map[i]]; output_index -= idx * output_strides[i]; } return input_index + output_index * input_strides[output_to_input_dim_map[0]]; - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = output_index / output_strides[i]; input_index += idx * input_strides[output_to_input_dim_map[i]]; @@ -164,12 +165,13 @@ static Index GetInputIndex(Index output_index, const array& outp template static array ComputeStrides(const array& sizes) { array strides; - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { strides[0] = 1; for (int i = 1; i < NumDims; ++i) { strides[i] = strides[i - 1] * sizes[i - 1]; } - } else { + } + else { strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { strides[i] = strides[i + 1] * sizes[i + 1]; @@ -226,7 +228,7 @@ static void test_uniform_block_shape() { // Test shape 'UniformAllDims' with larger 'max_coeff count' which spills // partially into first inner-most dimension. - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 7 * 5 * 5 * 5 * 5; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -236,7 +238,8 @@ static void test_uniform_block_shape() { VERIFY_IS_EQUAL(5, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } else { + } + else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 5 * 5 * 5 * 5 * 6; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -250,7 +253,7 @@ static void test_uniform_block_shape() { // Test shape 'UniformAllDims' with larger 'max_coeff count' which spills // fully into first inner-most dimension. - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 5 * 5 * 5 * 5; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -260,7 +263,8 @@ static void test_uniform_block_shape() { VERIFY_IS_EQUAL(5, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } else { + } + else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 5 * 5 * 5 * 5 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -274,7 +278,7 @@ static void test_uniform_block_shape() { // Test shape 'UniformAllDims' with larger 'max_coeff count' which spills // fully into first few inner-most dimensions. - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { DSizes dims(7, 5, 6, 17, 7); const Index max_coeff_count = 7 * 5 * 6 * 7 * 5; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -285,7 +289,8 @@ static void test_uniform_block_shape() { VERIFY_IS_EQUAL(7, block.dimensions()[3]); VERIFY_IS_EQUAL(5, block.dimensions()[4]); VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } else { + } + else { DSizes dims(7, 5, 6, 9, 7); const Index max_coeff_count = 5 * 5 * 5 * 6 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -299,7 +304,7 @@ static void test_uniform_block_shape() { } // Test shape 'UniformAllDims' with full allocation to all dims. - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { DSizes dims(7, 5, 6, 17, 7); const Index max_coeff_count = 7 * 5 * 6 * 17 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -310,7 +315,8 @@ static void test_uniform_block_shape() { VERIFY_IS_EQUAL(17, block.dimensions()[3]); VERIFY_IS_EQUAL(7, block.dimensions()[4]); VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } else { + } + else { DSizes dims(7, 5, 6, 9, 7); const Index max_coeff_count = 7 * 5 * 6 * 9 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -330,7 +336,7 @@ static void test_skewed_inner_dim_block_shape() { typedef internal::TensorBlockMapper<5, Layout> TensorBlockMapper; // Test shape 'SkewedInnerDims' with partial allocation to inner-most dim. - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 10 * 1 * 1 * 1 * 1; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -340,7 +346,8 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(1, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } else { + } + else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 1 * 1 * 1 * 1 * 6; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -353,7 +360,7 @@ static void test_skewed_inner_dim_block_shape() { } // Test shape 'SkewedInnerDims' with full allocation to inner-most dim. - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 1 * 1 * 1 * 1; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -363,7 +370,8 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(1, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } else { + } + else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 1 * 1 * 1 * 1 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -377,7 +385,7 @@ static void test_skewed_inner_dim_block_shape() { // Test shape 'SkewedInnerDims' with full allocation to inner-most dim, // and partial allocation to second inner-dim. - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 3 * 1 * 1 * 1; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -388,7 +396,8 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(1, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } else { + } + else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 1 * 1 * 1 * 15 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -403,7 +412,7 @@ static void test_skewed_inner_dim_block_shape() { // Test shape 'SkewedInnerDims' with full allocation to inner-most dim, // and partial allocation to third inner-dim. - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 5 * 5 * 1 * 1; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -415,7 +424,8 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(1, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } else { + } + else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 1 * 1 * 5 * 17 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -430,7 +440,7 @@ static void test_skewed_inner_dim_block_shape() { } // Test shape 'SkewedInnerDims' with full allocation to all dims. - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 5 * 6 * 17 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -441,7 +451,8 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(17, block.dimensions()[3]); VERIFY_IS_EQUAL(7, block.dimensions()[4]); VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } else { + } + else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 5 * 6 * 17 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); diff --git a/unsupported/test/tensor_block_eval.cpp b/unsupported/test/tensor_block_eval.cpp index 6b273c9c0..6c7acb7c4 100644 --- a/unsupported/test/tensor_block_eval.cpp +++ b/unsupported/test/tensor_block_eval.cpp @@ -86,14 +86,15 @@ static TensorBlockParams SkewedInnerBlock(DSizes dims) // Compute offsets for the first block coefficient. Index index = block.offset(); - if (static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / strides[i]; index -= idx * strides[i]; offsets[i] = idx; } if (NumDims > 0) offsets[0] = index; - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / strides[i]; index -= idx * strides[i]; @@ -287,7 +288,7 @@ static void test_eval_tensor_scan() { Tensor input(dims); input.setRandom(); - const Index axis = NumDims == 1 ? 0 : NumDims / 2; + constexpr Index axis = NumDims == 1 ? 0 : NumDims / 2; VerifyBlockEvaluator(input.cumsum(axis), [&dims]() { return RandomBlock(dims, 1, 5); }); @@ -351,7 +352,7 @@ static void test_eval_composed_block_ops() { padded_dims[i] = dims[i] + 3; } - const Index axis = NumDims == 1 ? 0 : NumDims / 2; + constexpr Index axis = NumDims == 1 ? 0 : NumDims / 2; Tensor input(dims); input.setRandom(); @@ -434,7 +435,7 @@ static void test_eval_tensor_select() { template static void test_eval_tensor_padding() { - const int inner_dim = Layout == static_cast(ColMajor) ? 0 : NumDims - 1; + constexpr int inner_dim = Layout == static_cast(ColMajor) ? 0 : NumDims - 1; DSizes dims = RandomDims(10, 20); Tensor input(dims); diff --git a/unsupported/test/tensor_block_io.cpp b/unsupported/test/tensor_block_io.cpp index c732502a7..42e65d129 100644 --- a/unsupported/test/tensor_block_io.cpp +++ b/unsupported/test/tensor_block_io.cpp @@ -38,14 +38,15 @@ template static Index GetInputIndex(Index output_index, const array& output_to_input_dim_map, const array& input_strides, const array& output_strides) { int input_index = 0; - if (Layout == ColMajor) { + EIGEN_IF_CONSTEXPR(Layout == ColMajor) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = output_index / output_strides[i]; input_index += idx * input_strides[output_to_input_dim_map[i]]; output_index -= idx * output_strides[i]; } return input_index + output_index * input_strides[output_to_input_dim_map[0]]; - } else { + } + else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = output_index / output_strides[i]; input_index += idx * input_strides[output_to_input_dim_map[i]];