From 7966ea495eb83181f32e25b1e3accc989afa5dfd Mon Sep 17 00:00:00 2001 From: Rasmus Munk Larsen <4643818-rmlarsen1@users.noreply.gitlab.com> Date: Sat, 6 Jun 2026 13:08:44 -0700 Subject: [PATCH] clang-format: register EIGEN_IF_CONSTEXPR as an IfMacro libeigen/eigen!2604 Co-authored-by: Rasmus Munk Larsen --- .clang-format | 2 + .../src/AccelerateSupport/AccelerateSupport.h | 23 +- Eigen/src/Cholesky/LDLT.h | 4 +- Eigen/src/CholmodSupport/CholmodSupport.h | 8 +- Eigen/src/Core/ConcatOp.h | 28 +- Eigen/src/Core/CoreEvaluators.h | 36 +- Eigen/src/Core/CwiseNullaryOp.h | 5 +- Eigen/src/Core/CwiseTernaryOp.h | 24 +- Eigen/src/Core/GenericPacketMath.h | 30 +- Eigen/src/Core/IO.h | 5 +- Eigen/src/Core/IndexedView.h | 8 +- Eigen/src/Core/MathFunctions.h | 2 +- Eigen/src/Core/PartialReduxEvaluator.h | 2 +- Eigen/src/Core/PlainObjectBase.h | 28 +- Eigen/src/Core/ProductEvaluators.h | 66 +- Eigen/src/Core/Ref.h | 5 +- Eigen/src/Core/Reshaped.h | 5 +- Eigen/src/Core/SelfAdjointView.h | 5 +- Eigen/src/Core/TriangularMatrix.h | 38 +- Eigen/src/Core/arch/AVX512/GemmKernel.h | 71 +- Eigen/src/Core/arch/AVX512/TrsmKernel.h | 354 +++++---- Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc | 177 +++-- Eigen/src/Core/arch/AltiVec/MatrixProduct.h | 594 ++++++++------- .../Core/arch/AltiVec/MatrixVectorProduct.inc | 709 +++++++++--------- Eigen/src/Core/arch/HVX/PacketMath.h | 7 +- .../Core/products/GeneralBlockPanelKernel.h | 192 ++--- .../products/GeneralMatrixMatrixTriangular.h | 12 +- Eigen/src/Core/products/GeneralMatrixVector.h | 12 +- .../Core/products/SelfadjointMatrixMatrix.h | 36 +- .../products/SelfadjointMatrixVector_BLAS.h | 5 +- Eigen/src/Core/products/SelfadjointProduct.h | 7 +- .../Core/products/SelfadjointRank2Update.h | 18 +- .../Core/products/TriangularMatrixMatrix.h | 18 +- .../Core/products/TriangularMatrixVector.h | 36 +- .../Core/products/TriangularSolverMatrix.h | 21 +- Eigen/src/Core/util/Memory.h | 14 +- Eigen/src/Geometry/OrthoMethods.h | 5 +- Eigen/src/Geometry/Transform.h | 2 +- .../BasicPreconditioners.h | 5 +- Eigen/src/SVD/JacobiSVD.h | 46 +- Eigen/src/SVD/SVDBase.h | 4 +- Eigen/src/SparseCholesky/SimplicialCholesky.h | 11 +- .../SparseCholesky/SimplicialCholesky_impl.h | 5 +- Eigen/src/SparseCore/SparseAssign.h | 6 +- Eigen/src/SparseCore/SparseBlock.h | 5 +- Eigen/src/SparseCore/SparseDenseProduct.h | 2 +- Eigen/src/SparseCore/SparseDiagonalProduct.h | 5 +- Eigen/src/SparseCore/SparseMatrixBase.h | 5 +- Eigen/src/SparseCore/SparseRef.h | 8 +- Eigen/src/SparseCore/SparseSelfAdjointView.h | 23 +- .../SparseSparseProductWithPruning.h | 5 +- Eigen/src/SparseCore/SparseTriangularView.h | 27 +- Eigen/src/SparseCore/SparsityPatternRef.h | 5 +- Eigen/src/SparseCore/TriangularSolver.h | 34 +- Eigen/src/SuperLUSupport/SuperLUSupport.h | 31 +- Eigen/src/ThreadPool/RunQueue.h | 5 +- blas/BandTriangularSolver.h | 4 +- blas/PackedTriangularMatrixVector.h | 8 +- blas/PackedTriangularSolverVector.h | 4 +- .../Eigen/src/AutoDiff/CoherentPadOp.h | 9 +- .../Eigen/src/SparseExtra/BlockSparseMatrix.h | 5 +- unsupported/Eigen/src/SparseExtra/MarketIO.h | 14 +- .../src/SparseExtra/MatrixMarketIterator.h | 10 +- unsupported/Eigen/src/Tensor/Tensor.h | 5 +- unsupported/Eigen/src/Tensor/TensorArgMax.h | 10 +- unsupported/Eigen/src/Tensor/TensorBlock.h | 37 +- .../Eigen/src/Tensor/TensorBroadcasting.h | 20 +- unsupported/Eigen/src/Tensor/TensorChipping.h | 10 +- .../Eigen/src/Tensor/TensorConcatenation.h | 35 +- .../Eigen/src/Tensor/TensorContraction.h | 7 +- .../src/Tensor/TensorContractionThreadPool.h | 5 +- .../Eigen/src/Tensor/TensorConvolution.h | 40 +- .../Eigen/src/Tensor/TensorEvaluator.h | 10 +- unsupported/Eigen/src/Tensor/TensorFFT.h | 10 +- .../Eigen/src/Tensor/TensorFixedSize.h | 5 +- .../Eigen/src/Tensor/TensorGenerator.h | 10 +- unsupported/Eigen/src/Tensor/TensorIO.h | 10 +- .../Eigen/src/Tensor/TensorImagePatch.h | 20 +- .../Eigen/src/Tensor/TensorInflation.h | 10 +- unsupported/Eigen/src/Tensor/TensorMap.h | 20 +- unsupported/Eigen/src/Tensor/TensorMorphing.h | 40 +- unsupported/Eigen/src/Tensor/TensorPadding.h | 19 +- unsupported/Eigen/src/Tensor/TensorPatch.h | 15 +- .../Eigen/src/Tensor/TensorReduction.h | 30 +- .../Eigen/src/Tensor/TensorReductionGpu.h | 13 +- unsupported/Eigen/src/Tensor/TensorRef.h | 10 +- unsupported/Eigen/src/Tensor/TensorReverse.h | 10 +- unsupported/Eigen/src/Tensor/TensorRoll.h | 15 +- unsupported/Eigen/src/Tensor/TensorScan.h | 5 +- .../Eigen/src/Tensor/TensorShuffling.h | 15 +- unsupported/Eigen/src/Tensor/TensorStriding.h | 20 +- unsupported/Eigen/src/Tensor/TensorTrace.h | 15 +- .../Eigen/src/Tensor/TensorVolumePatch.h | 20 +- unsupported/test/sparse_extra.cpp | 4 +- unsupported/test/tensor_block_access.cpp | 55 +- unsupported/test/tensor_block_eval.cpp | 10 +- unsupported/test/tensor_block_io.cpp | 5 +- 97 files changed, 1714 insertions(+), 1751 deletions(-) diff --git a/.clang-format b/.clang-format index 1f33cbed2..2022d3704 100644 --- a/.clang-format +++ b/.clang-format @@ -9,6 +9,8 @@ StatementMacros: - EIGEN_STATIC_ASSERT - EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED - EIGEN_INTERNAL_DENSE_STORAGE_CTOR_PLUGIN +IfMacros: + - EIGEN_IF_CONSTEXPR SortIncludes: false AttributeMacros: - EIGEN_STRONG_INLINE diff --git a/Eigen/src/AccelerateSupport/AccelerateSupport.h b/Eigen/src/AccelerateSupport/AccelerateSupport.h index 9739af48f..5a15ee694 100644 --- a/Eigen/src/AccelerateSupport/AccelerateSupport.h +++ b/Eigen/src/AccelerateSupport/AccelerateSupport.h @@ -176,35 +176,28 @@ class AccelerateImpl : public SparseSolverBase { mat.coeffRef(i, k) = numext::conj(mat.coeffRef(index_of_biggest_in_corner, i)); mat.coeffRef(index_of_biggest_in_corner, i) = numext::conj(tmp); } - EIGEN_IF_CONSTEXPR(NumTraits::IsComplex) - mat.coeffRef(index_of_biggest_in_corner, k) = numext::conj(mat.coeff(index_of_biggest_in_corner, k)); + EIGEN_IF_CONSTEXPR (NumTraits::IsComplex) + mat.coeffRef(index_of_biggest_in_corner, k) = numext::conj(mat.coeff(index_of_biggest_in_corner, k)); } // partition the matrix: diff --git a/Eigen/src/CholmodSupport/CholmodSupport.h b/Eigen/src/CholmodSupport/CholmodSupport.h index d5e9ae81c..6e81db3e3 100644 --- a/Eigen/src/CholmodSupport/CholmodSupport.h +++ b/Eigen/src/CholmodSupport/CholmodSupport.h @@ -83,11 +83,11 @@ cholmod_sparse viewAsCholmod(Ref res.dtype = 0; res.stype = -1; - EIGEN_IF_CONSTEXPR((std::is_same::value)) { res.itype = CHOLMOD_INT; } - else EIGEN_IF_CONSTEXPR((std::is_same::value)) { + EIGEN_IF_CONSTEXPR ((std::is_same::value)) { + res.itype = CHOLMOD_INT; + } else EIGEN_IF_CONSTEXPR ((std::is_same::value)) { res.itype = CHOLMOD_LONG; - } - else { + } else { eigen_assert(false && "Index type not supported yet"); } diff --git a/Eigen/src/Core/ConcatOp.h b/Eigen/src/Core/ConcatOp.h index 4ce3cb3b6..7cc780862 100644 --- a/Eigen/src/Core/ConcatOp.h +++ b/Eigen/src/Core/ConcatOp.h @@ -115,10 +115,9 @@ class Concat : public internal::dense_xpr_base> : evaluator_base> : evaluator_base EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { constexpr int packetSize = unpacket_traits::size; - EIGEN_IF_CONSTEXPR(Direction == Vertical) { + EIGEN_IF_CONSTEXPR (Direction == Vertical) { const Index boundary = m_lhsRows.value(); if (row >= boundary) return m_rhsImpl.template packet(row - boundary, col); // Column-major: inner=rows, packet extends along rows and may straddle the row boundary. // Row-major: inner=cols, packet extends along cols — never crosses the row boundary. - EIGEN_IF_CONSTEXPR(!IsRowMajor) { + EIGEN_IF_CONSTEXPR (!IsRowMajor) { if (row + packetSize > boundary) return packetBoundary(row, col); } return m_lhsImpl.template packet(row, col); - } - else { + } else { const Index boundary = m_lhsCols.value(); if (col >= boundary) return m_rhsImpl.template packet(row, col - boundary); // Row-major: inner=cols, packet extends along cols and may straddle the col boundary. // Column-major: inner=rows, packet extends along rows — never crosses the col boundary. - EIGEN_IF_CONSTEXPR(IsRowMajor) { + EIGEN_IF_CONSTEXPR (IsRowMajor) { if (col + packetSize > boundary) return packetBoundary(row, col); } return m_lhsImpl.template packet(row, col); @@ -233,20 +230,19 @@ struct evaluator> : evaluator_base EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index row, Index col, Index begin, Index count) const { - EIGEN_IF_CONSTEXPR(Direction == Vertical) { + EIGEN_IF_CONSTEXPR (Direction == Vertical) { const Index boundary = m_lhsRows.value(); if (row >= boundary) return m_rhsImpl.template packetSegment(row - boundary, col, begin, count); - EIGEN_IF_CONSTEXPR(!IsRowMajor) { + EIGEN_IF_CONSTEXPR (!IsRowMajor) { if (row + begin + count > boundary) return packetSegmentBoundary(row, col, begin, count); } return m_lhsImpl.template packetSegment(row, col, begin, count); - } - else { + } else { const Index boundary = m_lhsCols.value(); if (col >= boundary) return m_rhsImpl.template packetSegment(row, col - boundary, begin, count); - EIGEN_IF_CONSTEXPR(IsRowMajor) { + EIGEN_IF_CONSTEXPR (IsRowMajor) { if (col + begin + count > boundary) return packetSegmentBoundary(row, col, begin, count); } return m_lhsImpl.template packetSegment(row, col, begin, count); diff --git a/Eigen/src/Core/CoreEvaluators.h b/Eigen/src/Core/CoreEvaluators.h index 53f39da2a..1fdda4268 100644 --- a/Eigen/src/Core/CoreEvaluators.h +++ b/Eigen/src/Core/CoreEvaluators.h @@ -1429,9 +1429,10 @@ struct unary_evaluator, IndexBa template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { - EIGEN_IF_CONSTEXPR(ForwardLinearAccess) - return m_argImpl.template packet(m_linear_offset.value() + index); - else return packet(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); + EIGEN_IF_CONSTEXPR (ForwardLinearAccess) + return m_argImpl.template packet(m_linear_offset.value() + index); + else + return packet(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); } template @@ -1441,10 +1442,11 @@ struct unary_evaluator, IndexBa template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { - EIGEN_IF_CONSTEXPR(ForwardLinearAccess) - return m_argImpl.template writePacket(m_linear_offset.value() + index, x); - else return writePacket(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0, x); + EIGEN_IF_CONSTEXPR (ForwardLinearAccess) + return m_argImpl.template writePacket(m_linear_offset.value() + index, x); + else + return writePacket(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0, + x); } template @@ -1455,10 +1457,11 @@ struct unary_evaluator, IndexBa template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index index, Index begin, Index count) const { - EIGEN_IF_CONSTEXPR(ForwardLinearAccess) - return m_argImpl.template packetSegment(m_linear_offset.value() + index, begin, count); - else return packetSegment(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0, begin, count); + EIGEN_IF_CONSTEXPR (ForwardLinearAccess) + return m_argImpl.template packetSegment(m_linear_offset.value() + index, begin, count); + else + return packetSegment(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0, + begin, count); } template @@ -1471,11 +1474,12 @@ struct unary_evaluator, IndexBa template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacketSegment(Index index, const PacketType& x, Index begin, Index count) { - EIGEN_IF_CONSTEXPR(ForwardLinearAccess) - return m_argImpl.template writePacketSegment(m_linear_offset.value() + index, x, begin, - count); - else return writePacketSegment(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0, x, begin, count); + EIGEN_IF_CONSTEXPR (ForwardLinearAccess) + return m_argImpl.template writePacketSegment(m_linear_offset.value() + index, x, begin, + count); + else + return writePacketSegment(RowsAtCompileTime == 1 ? 0 : index, + RowsAtCompileTime == 1 ? index : 0, x, begin, count); } protected: diff --git a/Eigen/src/Core/CwiseNullaryOp.h b/Eigen/src/Core/CwiseNullaryOp.h index 0241276ec..b9ed71102 100644 --- a/Eigen/src/Core/CwiseNullaryOp.h +++ b/Eigen/src/Core/CwiseNullaryOp.h @@ -142,8 +142,9 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE #endif DenseBase::NullaryExpr(Index size, const CustomNullaryOp& func) { EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) - EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { return CwiseNullaryOp(1, size, func); } - else { + EIGEN_IF_CONSTEXPR (RowsAtCompileTime == 1) { + return CwiseNullaryOp(1, size, func); + } else { return CwiseNullaryOp(size, 1, func); } } diff --git a/Eigen/src/Core/CwiseTernaryOp.h b/Eigen/src/Core/CwiseTernaryOp.h index 0a33cdb1c..08852ee42 100644 --- a/Eigen/src/Core/CwiseTernaryOp.h +++ b/Eigen/src/Core/CwiseTernaryOp.h @@ -122,30 +122,26 @@ class CwiseTernaryOp : public CwiseTernaryOpImpl>::RowsAtCompileTime == Dynamic && - internal::traits>::RowsAtCompileTime == Dynamic) { + EIGEN_IF_CONSTEXPR (internal::traits>::RowsAtCompileTime == Dynamic && + internal::traits>::RowsAtCompileTime == Dynamic) { return m_arg3.rows(); - } - else EIGEN_IF_CONSTEXPR(internal::traits>::RowsAtCompileTime == Dynamic && - internal::traits>::RowsAtCompileTime == Dynamic) { + } else EIGEN_IF_CONSTEXPR (internal::traits>::RowsAtCompileTime == Dynamic && + internal::traits>::RowsAtCompileTime == Dynamic) { return m_arg2.rows(); - } - else { + } else { return m_arg1.rows(); } } EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Index cols() const { // return the fixed size type if available to enable compile time // optimizations - EIGEN_IF_CONSTEXPR(internal::traits>::ColsAtCompileTime == Dynamic && - internal::traits>::ColsAtCompileTime == Dynamic) { + EIGEN_IF_CONSTEXPR (internal::traits>::ColsAtCompileTime == Dynamic && + internal::traits>::ColsAtCompileTime == Dynamic) { return m_arg3.cols(); - } - else EIGEN_IF_CONSTEXPR(internal::traits>::ColsAtCompileTime == Dynamic && - internal::traits>::ColsAtCompileTime == Dynamic) { + } else EIGEN_IF_CONSTEXPR (internal::traits>::ColsAtCompileTime == Dynamic && + internal::traits>::ColsAtCompileTime == Dynamic) { return m_arg2.cols(); - } - else { + } else { return m_arg1.cols(); } } diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index 3877ba379..06e7a8937 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h @@ -1448,8 +1448,9 @@ inline void pstore1(typename unpacket_traits::type* to, const typename u * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_traits::type* from) { - EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits::alignment) { return pload(from); } - else { + EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits::alignment) { + return pload(from); + } else { return ploadu(from); } } @@ -1459,8 +1460,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_trai template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(const typename unpacket_traits::type* from, const Index n, const Index offset = 0) { - EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits::alignment) { return pload_partial(from, n, offset); } - else { + EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits::alignment) { + return pload_partial(from, n, offset); + } else { return ploadu_partial(from, n, offset); } } @@ -1469,8 +1471,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(const typename unpac * The pointer \a from must be aligned on a \a Alignment bytes boundary. */ template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& from) { - EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits::alignment) { pstore(to, from); } - else { + EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits::alignment) { + pstore(to, from); + } else { pstoreu(to, from); } } @@ -1480,8 +1483,9 @@ EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& fro template EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret_partial(Scalar* to, const Packet& from, const Index n, const Index offset = 0) { - EIGEN_IF_CONSTEXPR(Alignment >= unpacket_traits::alignment) { pstore_partial(to, from, n, offset); } - else { + EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits::alignment) { + pstore_partial(to, from, n, offset); + } else { pstoreu_partial(to, from, n, offset); } } @@ -1677,8 +1681,9 @@ template EIGEN_DEVICE_FUNC inline Packet ploadtSegment(const typename unpacket_traits::type* from, Index begin, Index count) { constexpr int RequiredAlignment = unpacket_traits::alignment; - EIGEN_IF_CONSTEXPR(Alignment >= RequiredAlignment) { return ploadSegment(from, begin, count); } - else { + EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) { + return ploadSegment(from, begin, count); + } else { return ploaduSegment(from, begin, count); } } @@ -1688,8 +1693,9 @@ Elements outside of the range [begin, begin + count) are not defined.*/ template EIGEN_DEVICE_FUNC inline void pstoretSegment(Scalar* to, const Packet& from, Index begin, Index count) { constexpr int RequiredAlignment = unpacket_traits::alignment; - EIGEN_IF_CONSTEXPR(Alignment >= RequiredAlignment) { pstoreSegment(to, from, begin, count); } - else { + EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) { + pstoreSegment(to, from, begin, count); + } else { pstoreuSegment(to, from, begin, count); } } diff --git a/Eigen/src/Core/IO.h b/Eigen/src/Core/IO.h index faeeb9f5b..0111acf80 100644 --- a/Eigen/src/Core/IO.h +++ b/Eigen/src/Core/IO.h @@ -150,8 +150,9 @@ std::ostream& print_matrix(std::ostream& s, const Derived& _m, const IOFormat& f if (fmt.precision == StreamPrecision) { explicit_precision = 0; } else if (fmt.precision == FullPrecision) { - EIGEN_IF_CONSTEXPR(NumTraits::IsInteger) { explicit_precision = 0; } - else { + EIGEN_IF_CONSTEXPR (NumTraits::IsInteger) { + explicit_precision = 0; + } else { explicit_precision = significant_decimals_impl::run(); } } else { diff --git a/Eigen/src/Core/IndexedView.h b/Eigen/src/Core/IndexedView.h index 1ecaf9ad9..16ba46a73 100644 --- a/Eigen/src/Core/IndexedView.h +++ b/Eigen/src/Core/IndexedView.h @@ -199,13 +199,13 @@ class IndexedViewImpl IndexedViewImpl(XprType& xpr, const T0& rowIndices, const T1& colIndices) : Base(xpr, rowIndices, colIndices) {} Index rowIncrement() const { - EIGEN_IF_CONSTEXPR(traits::RowIncr != DynamicIndex && traits::RowIncr != Undefined) { + EIGEN_IF_CONSTEXPR (traits::RowIncr != DynamicIndex && traits::RowIncr != Undefined) { return traits::RowIncr; } return IndexedViewHelper::incr(this->rowIndices()); } Index colIncrement() const { - EIGEN_IF_CONSTEXPR(traits::ColIncr != DynamicIndex && traits::ColIncr != Undefined) { + EIGEN_IF_CONSTEXPR (traits::ColIncr != DynamicIndex && traits::ColIncr != Undefined) { return traits::ColIncr; } return IndexedViewHelper::incr(this->colIndices()); @@ -228,14 +228,14 @@ class IndexedViewImpl } EIGEN_DEVICE_FUNC constexpr Index innerStride() const noexcept { - EIGEN_IF_CONSTEXPR(traits::InnerStrideAtCompileTime != Dynamic) { + EIGEN_IF_CONSTEXPR (traits::InnerStrideAtCompileTime != Dynamic) { return traits::InnerStrideAtCompileTime; } return innerIncrement() * this->nestedExpression().innerStride(); } EIGEN_DEVICE_FUNC constexpr Index outerStride() const noexcept { - EIGEN_IF_CONSTEXPR(traits::OuterStrideAtCompileTime != Dynamic) { + EIGEN_IF_CONSTEXPR (traits::OuterStrideAtCompileTime != Dynamic) { return traits::OuterStrideAtCompileTime; } return outerIncrement() * this->nestedExpression().outerStride(); diff --git a/Eigen/src/Core/MathFunctions.h b/Eigen/src/Core/MathFunctions.h index 783a283aa..477cf3f42 100644 --- a/Eigen/src/Core/MathFunctions.h +++ b/Eigen/src/Core/MathFunctions.h @@ -921,7 +921,7 @@ struct copysign_impl { template struct copysign_impl { EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& a, const Scalar& b) { - EIGEN_IF_CONSTEXPR(!NumTraits::IsSigned) return a; + EIGEN_IF_CONSTEXPR (!NumTraits::IsSigned) return a; const Scalar abs_a = a < Scalar(0) ? -a : a; return b < Scalar(0) ? -abs_a : abs_a; } diff --git a/Eigen/src/Core/PartialReduxEvaluator.h b/Eigen/src/Core/PartialReduxEvaluator.h index 8b97c1faf..3cf597ecb 100644 --- a/Eigen/src/Core/PartialReduxEvaluator.h +++ b/Eigen/src/Core/PartialReduxEvaluator.h @@ -203,7 +203,7 @@ struct evaluator > // Workaround for issue 1612 (closed): when PacketSize==1 (i.e. complex with 128bits registers) the // storage-order of panel gets reversed and methods like packetByOuterInner do not make sense in this context, so // bypass "vectorization": - EIGEN_IF_CONSTEXPR(PacketSize == 1) return internal::pset1(coeff(idx)); + EIGEN_IF_CONSTEXPR (PacketSize == 1) return internal::pset1(coeff(idx)); Index startRow = Direction == Vertical ? 0 : idx; Index startCol = Direction == Vertical ? idx : 0; diff --git a/Eigen/src/Core/PlainObjectBase.h b/Eigen/src/Core/PlainObjectBase.h index 1963fd023..77176cfe8 100644 --- a/Eigen/src/Core/PlainObjectBase.h +++ b/Eigen/src/Core/PlainObjectBase.h @@ -171,8 +171,9 @@ class PlainObjectBase : public internal::dense_xpr_base::type { * * See DenseCoeffsBase::coeff(Index) const for details. */ EIGEN_DEVICE_FUNC constexpr const Scalar& coeff(Index rowId, Index colId) const { - EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; } - else { // column-major + EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) { + return m_storage.data()[colId + rowId * m_storage.cols()]; + } else { // column-major return m_storage.data()[rowId + colId * m_storage.rows()]; } } @@ -188,8 +189,9 @@ class PlainObjectBase : public internal::dense_xpr_base::type { * * See DenseCoeffsBase::coeffRef(Index,Index) const for details. */ EIGEN_DEVICE_FUNC constexpr Scalar& coeffRef(Index rowId, Index colId) { - EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; } - else { // column-major + EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) { + return m_storage.data()[colId + rowId * m_storage.cols()]; + } else { // column-major return m_storage.data()[rowId + colId * m_storage.rows()]; } } @@ -203,8 +205,9 @@ class PlainObjectBase : public internal::dense_xpr_base::type { /** This is the const version of coeffRef(Index,Index) which is thus synonym of coeff(Index,Index). * It is provided for convenience. */ EIGEN_DEVICE_FUNC constexpr const Scalar& coeffRef(Index rowId, Index colId) const { - EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { return m_storage.data()[colId + rowId * m_storage.cols()]; } - else { // column-major + EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) { + return m_storage.data()[colId + rowId * m_storage.cols()]; + } else { // column-major return m_storage.data()[rowId + colId * m_storage.rows()]; } } @@ -303,8 +306,9 @@ class PlainObjectBase : public internal::dense_xpr_base::type { #ifdef EIGEN_INITIALIZE_COEFFS bool size_changed = size != this->size(); #endif - EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { m_storage.resize(size, 1, size); } - else { + EIGEN_IF_CONSTEXPR (RowsAtCompileTime == 1) { + m_storage.resize(size, 1, size); + } else { m_storage.resize(size, size, 1); } #ifdef EIGEN_INITIALIZE_COEFFS @@ -347,15 +351,13 @@ class PlainObjectBase : public internal::dense_xpr_base::type { other.rows(), other.cols()); #endif const Index othersize = other.rows() * other.cols(); - EIGEN_IF_CONSTEXPR(RowsAtCompileTime == 1) { + EIGEN_IF_CONSTEXPR (RowsAtCompileTime == 1) { eigen_assert(other.rows() == 1 || other.cols() == 1); resize(1, othersize); - } - else EIGEN_IF_CONSTEXPR(ColsAtCompileTime == 1) { + } else EIGEN_IF_CONSTEXPR (ColsAtCompileTime == 1) { eigen_assert(other.rows() == 1 || other.cols() == 1); resize(othersize, 1); - } - else { + } else { resize(other.rows(), other.cols()); } } diff --git a/Eigen/src/Core/ProductEvaluators.h b/Eigen/src/Core/ProductEvaluators.h index 7af29111e..d7ed14942 100644 --- a/Eigen/src/Core/ProductEvaluators.h +++ b/Eigen/src/Core/ProductEvaluators.h @@ -899,16 +899,15 @@ struct triangular_diagonal_product_impl { const Index rows = matrix.rows(); const Index cols = matrix.cols(); for (Index col = 0; col < cols; ++col) { - EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) { + EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) { const Index end = (std::min)(rows, ((Mode & (UnitDiag | ZeroDiag)) ? col : col + 1)); addStoredSegment(dst, matrix, diagonal, 0, end, col, alpha); - } - else { + } else { const Index begin = ((Mode & (UnitDiag | ZeroDiag)) ? col + 1 : col); addStoredSegment(dst, matrix, diagonal, begin, rows - begin, col, alpha); } - EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) { + EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) { if (col < rows) addUnitCoeff(dst, diagonal, col, alpha); } } @@ -928,10 +927,9 @@ struct triangular_diagonal_product_impl { template static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addUnitCoeff(Dest& dst, const DiagonalType& diagonal, Index index, const Alpha& alpha) { - EIGEN_IF_CONSTEXPR(ProductOrder == OnTheLeft) { + EIGEN_IF_CONSTEXPR (ProductOrder == OnTheLeft) { dst.coeffRef(index, index) += alpha * (diagonal.coeff(index) * MatrixScalar(1)); - } - else { + } else { dst.coeffRef(index, index) += alpha * (MatrixScalar(1) * diagonal.coeff(index)); } } @@ -1041,10 +1039,9 @@ struct selfadjoint_diagonal_product_impl { // Stored half: one column-strided segment per output column. for (Index col = 0; col < size; ++col) { - EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) { + EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) { storedSegment(dst, matrix, diagonal, 0, col + 1, col, alpha); - } - else { + } else { storedSegment(dst, matrix, diagonal, col, size - col, col, alpha); } } @@ -1053,7 +1050,7 @@ struct selfadjoint_diagonal_product_impl { for (Index ib = 0; ib < size; ib += BlockSize) { const Index ib_end = numext::mini(size, ib + BlockSize); const Index br = ib_end - ib; - EIGEN_IF_CONSTEXPR((Mode & Upper) == Upper) { + EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) { // Off-diagonal: write strict-lower of dst from strict-upper of source. for (Index jb = 0; jb < ib; jb += BlockSize) { const Index bc = numext::mini(jb + BlockSize, ib) - jb; @@ -1062,8 +1059,7 @@ struct selfadjoint_diagonal_product_impl { // Diagonal tile: in-tile strict-lower mirror. for (Index col = ib; col < ib_end; ++col) conjugateSegment(dst, matrix, diagonal, col + 1, ib_end - col - 1, col, alpha); - } - else { + } else { // Off-diagonal: write strict-upper of dst from strict-lower of source. for (Index jb = ib_end; jb < size; jb += BlockSize) { const Index bc = numext::mini(size, jb + BlockSize) - jb; @@ -1083,10 +1079,9 @@ struct selfadjoint_diagonal_product_impl { if (size <= 0) return; auto dstSegment = dst.col(col).segment(begin, size); auto srcSegment = matrix.col(col).segment(begin, size); - EIGEN_IF_CONSTEXPR(Accumulate) { + EIGEN_IF_CONSTEXPR (Accumulate) { diagonal_product_segment_impl::run(dstSegment, srcSegment, diagonal, begin, col, alpha); - } - else { + } else { diagonal_product_segment_impl::runOverwrite(dstSegment, srcSegment, diagonal, begin, col); } } @@ -1098,10 +1093,9 @@ struct selfadjoint_diagonal_product_impl { if (size <= 0) return; auto dstSegment = dst.col(col).segment(begin, size); auto srcSegment = matrix.row(col).segment(begin, size).conjugate().transpose(); - EIGEN_IF_CONSTEXPR(Accumulate) { + EIGEN_IF_CONSTEXPR (Accumulate) { diagonal_product_segment_impl::run(dstSegment, srcSegment, diagonal, begin, col, alpha); - } - else { + } else { diagonal_product_segment_impl::runOverwrite(dstSegment, srcSegment, diagonal, begin, col); } } @@ -1115,17 +1109,18 @@ struct selfadjoint_diagonal_product_impl { Index br, Index bc, const Alpha& alpha) { auto dstBlock = dst.block(ib, jb, br, bc); auto srcAdjoint = matrix.block(jb, ib, bc, br).adjoint(); - EIGEN_IF_CONSTEXPR(ProductOrder == OnTheRight) { + EIGEN_IF_CONSTEXPR (ProductOrder == OnTheRight) { auto scaled = srcAdjoint * diagonal.segment(jb, bc).asDiagonal(); - EIGEN_IF_CONSTEXPR(Accumulate) { dstBlock.noalias() += alpha * scaled; } - else { + EIGEN_IF_CONSTEXPR (Accumulate) { + dstBlock.noalias() += alpha * scaled; + } else { dstBlock.noalias() = scaled; } - } - else { + } else { auto scaled = diagonal.segment(ib, br).asDiagonal() * srcAdjoint; - EIGEN_IF_CONSTEXPR(Accumulate) { dstBlock.noalias() += alpha * scaled; } - else { + EIGEN_IF_CONSTEXPR (Accumulate) { + dstBlock.noalias() += alpha * scaled; + } else { dstBlock.noalias() = scaled; } } @@ -1180,13 +1175,12 @@ struct generic_product_impl static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) { - EIGEN_IF_CONSTEXPR(LhsBlasTraits::HasScalarFactor) { + EIGEN_IF_CONSTEXPR (LhsBlasTraits::HasScalarFactor) { // Folded scalar factor present: zero dst then accumulate at the extracted alpha. Scalar factor = LhsBlasTraits::extractScalarFactor(lhs.nestedExpression()); dst.setZero(); Kernel::run(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal(), factor); - } - else { + } else { // No scalar factor: kernel writes every entry exactly once, skip setZero. Kernel::runOverwrite(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal()); } @@ -1219,12 +1213,11 @@ struct generic_product_impl static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) { - EIGEN_IF_CONSTEXPR(RhsBlasTraits::HasScalarFactor) { + EIGEN_IF_CONSTEXPR (RhsBlasTraits::HasScalarFactor) { Scalar factor = RhsBlasTraits::extractScalarFactor(rhs.nestedExpression()); dst.setZero(); Kernel::run(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal(), factor); - } - else { + } else { Kernel::runOverwrite(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal()); } } @@ -1471,11 +1464,11 @@ struct triangular_diagonal_product_lazy_evaluator_base : evaluator_base const bool inActive = ((Mode & Upper) == Upper) ? (row <= col) : (row >= col); if (!inActive) return Scalar(0); if (row == col) { - EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) { + EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) { return ProductOrder == OnTheLeft ? Scalar(m_diagImpl.coeff(row) * MatrixScalar(1)) : Scalar(MatrixScalar(1) * m_diagImpl.coeff(col)); } - EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag) return Scalar(0); + EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag) return Scalar(0); } return ProductOrder == OnTheLeft ? Scalar(m_diagImpl.coeff(row) * m_matImpl.coeff(row, col)) : Scalar(m_matImpl.coeff(row, col) * m_diagImpl.coeff(col)); @@ -1701,8 +1694,9 @@ struct transposition_matrix_product { for (Index k = (Transposed ? size - 1 : 0); Transposed ? k >= 0 : k < size; Transposed ? --k : ++k) if (Index(j = tr.coeff(k)) != k) { - EIGEN_IF_CONSTEXPR(Side == OnTheLeft) { dst.row(k).swap(dst.row(j)); } - else EIGEN_IF_CONSTEXPR(Side == OnTheRight) { + EIGEN_IF_CONSTEXPR (Side == OnTheLeft) { + dst.row(k).swap(dst.row(j)); + } else EIGEN_IF_CONSTEXPR (Side == OnTheRight) { dst.col(k).swap(dst.col(j)); } } diff --git a/Eigen/src/Core/Ref.h b/Eigen/src/Core/Ref.h index b39863049..74d4c042c 100644 --- a/Eigen/src/Core/Ref.h +++ b/Eigen/src/Core/Ref.h @@ -126,12 +126,11 @@ class RefBase : public MapBase { // Determine runtime rows and columns. Index rows = expr.rows(); Index cols = expr.cols(); - EIGEN_IF_CONSTEXPR(PlainObjectType::RowsAtCompileTime == 1) { + EIGEN_IF_CONSTEXPR (PlainObjectType::RowsAtCompileTime == 1) { eigen_assert(expr.rows() == 1 || expr.cols() == 1); rows = 1; cols = expr.size(); - } - else EIGEN_IF_CONSTEXPR(PlainObjectType::ColsAtCompileTime == 1) { + } else EIGEN_IF_CONSTEXPR (PlainObjectType::ColsAtCompileTime == 1) { eigen_assert(expr.rows() == 1 || expr.cols() == 1); rows = expr.size(); cols = 1; diff --git a/Eigen/src/Core/Reshaped.h b/Eigen/src/Core/Reshaped.h index c9e75186d..66a04bffc 100644 --- a/Eigen/src/Core/Reshaped.h +++ b/Eigen/src/Core/Reshaped.h @@ -296,11 +296,10 @@ struct reshaped_evaluator RowCol; EIGEN_DEVICE_FUNC constexpr inline RowCol index_remap(Index rowId, Index colId) const { - EIGEN_IF_CONSTEXPR(Order == ColMajor) { + EIGEN_IF_CONSTEXPR (Order == ColMajor) { const Index nth_elem_idx = colId * m_xpr.rows() + rowId; return RowCol(nth_elem_idx % m_xpr.nestedExpression().rows(), nth_elem_idx / m_xpr.nestedExpression().rows()); - } - else { + } else { const Index nth_elem_idx = colId + rowId * m_xpr.cols(); return RowCol(nth_elem_idx / m_xpr.nestedExpression().cols(), nth_elem_idx % m_xpr.nestedExpression().cols()); } diff --git a/Eigen/src/Core/SelfAdjointView.h b/Eigen/src/Core/SelfAdjointView.h index cb076b73a..558a9aa34 100644 --- a/Eigen/src/Core/SelfAdjointView.h +++ b/Eigen/src/Core/SelfAdjointView.h @@ -219,11 +219,10 @@ class SelfAdjointView : public TriangularBase const Index n = m_matrix.rows(); for (Index col = 0; col < n; ++col) { RealScalar_ abs_col_sum; - EIGEN_IF_CONSTEXPR(UpLo == Lower) { + EIGEN_IF_CONSTEXPR (UpLo == Lower) { abs_col_sum = m_matrix.col(col).tail(n - col).template lpNorm<1>() + m_matrix.row(col).head(col).template lpNorm<1>(); - } - else { + } else { abs_col_sum = m_matrix.col(col).head(col).template lpNorm<1>() + m_matrix.row(col).tail(n - col).template lpNorm<1>(); } diff --git a/Eigen/src/Core/TriangularMatrix.h b/Eigen/src/Core/TriangularMatrix.h index a4fbf98a5..136c99133 100644 --- a/Eigen/src/Core/TriangularMatrix.h +++ b/Eigen/src/Core/TriangularMatrix.h @@ -391,11 +391,11 @@ class TriangularView /** \returns the determinant of the triangular matrix * \sa MatrixBase::determinant() */ EIGEN_DEVICE_FUNC Scalar determinant() const { - EIGEN_IF_CONSTEXPR(Mode & UnitDiag) { return 1; } - else EIGEN_IF_CONSTEXPR(Mode & ZeroDiag) { + EIGEN_IF_CONSTEXPR (Mode & UnitDiag) { + return 1; + } else EIGEN_IF_CONSTEXPR (Mode & ZeroDiag) { return 0; - } - else { + } else { return m_matrix.diagonal().prod(); } } @@ -777,18 +777,20 @@ class triangular_dense_assignment_kernel #endif EIGEN_DEVICE_FUNC void assignDiagonalCoeff(Index id) { - EIGEN_IF_CONSTEXPR(Mode == UnitDiag && SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(1)); } - else EIGEN_IF_CONSTEXPR(Mode == ZeroDiag && SetOpposite) { + EIGEN_IF_CONSTEXPR (Mode == UnitDiag && SetOpposite) { + m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(1)); + } else EIGEN_IF_CONSTEXPR (Mode == ZeroDiag && SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(id, id), Scalar(0)); - } - else EIGEN_IF_CONSTEXPR(Mode == 0) { + } else EIGEN_IF_CONSTEXPR (Mode == 0) { Base::assignCoeff(id, id); } } EIGEN_DEVICE_FUNC void assignOppositeCoeff(Index row, Index col) { eigen_internal_assert(row != col); - EIGEN_IF_CONSTEXPR(SetOpposite) { m_functor.assignCoeff(m_dst.coeffRef(row, col), Scalar(0)); } + EIGEN_IF_CONSTEXPR (SetOpposite) { + m_functor.assignCoeff(m_dst.coeffRef(row, col), Scalar(0)); + } } }; @@ -894,8 +896,9 @@ struct triangular_assignment_loop { kernel.assignDiagonalCoeff(row); else if (((Mode & Lower) && row > col) || ((Mode & Upper) && row < col)) kernel.assignCoeff(row, col); - else - EIGEN_IF_CONSTEXPR(SetOpposite) { kernel.assignOppositeCoeff(row, col); } + else EIGEN_IF_CONSTEXPR (SetOpposite) { + kernel.assignOppositeCoeff(row, col); + } } }; @@ -942,22 +945,19 @@ struct triangular_assignment_loop { const Index maxi = numext::mini(outer, innerSize); Index i = 0; - EIGEN_IF_CONSTEXPR(ActiveBeforeDiag) { + EIGEN_IF_CONSTEXPR (ActiveBeforeDiag) { for (; i < maxi; ++i) kernel.assignCoeff(row(outer, i), col(outer, i)); - } - else EIGEN_IF_CONSTEXPR(SetOpposite) { + } else EIGEN_IF_CONSTEXPR (SetOpposite) { for (; i < maxi; ++i) kernel.assignOppositeCoeff(row(outer, i), col(outer, i)); - } - else { + } else { i = maxi; } if (i < innerSize) kernel.assignDiagonalCoeff(i++); - EIGEN_IF_CONSTEXPR(!ActiveBeforeDiag) { + EIGEN_IF_CONSTEXPR (!ActiveBeforeDiag) { for (; i < innerSize; ++i) kernel.assignCoeff(row(outer, i), col(outer, i)); - } - else EIGEN_IF_CONSTEXPR(SetOpposite) { + } else EIGEN_IF_CONSTEXPR (SetOpposite) { for (; i < innerSize; ++i) kernel.assignOppositeCoeff(row(outer, i), col(outer, i)); } } diff --git a/Eigen/src/Core/arch/AVX512/GemmKernel.h b/Eigen/src/Core/arch/AVX512/GemmKernel.h index 6855ea8fe..89272b40f 100644 --- a/Eigen/src/Core/arch/AVX512/GemmKernel.h +++ b/Eigen/src/Core/arch/AVX512/GemmKernel.h @@ -1001,10 +1001,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= 4 ? (cols / 4) * 4 : 0; Index count = 0; const Index peeled_k = (depth / PacketSize) * PacketSize; - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { // skip what we have before - EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset; const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); @@ -1014,7 +1014,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs kernel; @@ -1027,7 +1027,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs(k); kernel.packet[7] = dm7.template loadPacket(k); - EIGEN_IF_CONSTEXPR(PacketSize == 4) { + EIGEN_IF_CONSTEXPR (PacketSize == 4) { // For PacketSize==4 we cannot ptranspose 8 packets directly; compose two // 4-packet transposes (cols 0-3 and 4-7) and interleave the halves so // the 8 stores produce 4 rows of 8 packed elements. @@ -1051,8 +1051,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= 4) { + EIGEN_IF_CONSTEXPR (nr >= 4) { for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { // skip what we have before - EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset; const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3); Index k = 0; - EIGEN_IF_CONSTEXPR((PacketSize % 4) == 0 || PacketSize == 2) { + EIGEN_IF_CONSTEXPR ((PacketSize % 4) == 0 || PacketSize == 2) { for (; k < peeled_k; k += PacketSize) { PacketBlock kernel; kernel.packet[0] = dm0.template loadPacket(k); kernel.packet[1] = dm1.template loadPacket(k); kernel.packet[2] = dm2.template loadPacket(k); kernel.packet[3] = dm3.template loadPacket(k); - EIGEN_IF_CONSTEXPR(PacketSize == 2) { + EIGEN_IF_CONSTEXPR (PacketSize == 2) { // See the matching note in GeneralBlockPanelKernel.h. PacketBlock tmp01; tmp01.packet[0] = kernel.packet[0]; @@ -1114,8 +1113,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= 4 ? (cols / 4) * 4 : 0; Index count = 0; - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { // skip what we have before - EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset; for (Index k = 0; k < depth; k++) { - EIGEN_IF_CONSTEXPR(PacketSize == 8) { + EIGEN_IF_CONSTEXPR (PacketSize == 8) { // Packet A = ploadu(&rhs.data()[k*rhs.stride() + j2]); Packet A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); - } - else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 8) { + } else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 8) { HalfPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); - } - else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 8) { + } else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 8) { QuarterPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); - } - else EIGEN_IF_CONSTEXPR(PacketSize == 4) { + } else EIGEN_IF_CONSTEXPR (PacketSize == 4) { // Packet A = ploadu(&rhs.data()[k*rhs.stride() + j2]); // Packet B = ploadu(&rhs.data()[k*rhs.stride() + j2 + PacketSize]); Packet A = rhs.template loadPacket(k, j2); Packet B = rhs.template loadPacket(k, j2 + PacketSize); pstoreu(blockB + count, cj.pconj(A)); pstoreu(blockB + count + PacketSize, cj.pconj(B)); - } - else { + } else { // const Scalar* b0 = &rhs.data()[k*rhs.stride() + j2]; const LinearMapper dm0 = rhs.getLinearMapper(k, j2); blockB[count + 0] = cj(dm0(0)); @@ -1215,31 +1209,28 @@ struct gemm_pack_rhs= 4) { + EIGEN_IF_CONSTEXPR (nr >= 4) { for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { // skip what we have before - EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset; for (Index k = 0; k < depth; k++) { - EIGEN_IF_CONSTEXPR(PacketSize == 4) { + EIGEN_IF_CONSTEXPR (PacketSize == 4) { Packet A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += PacketSize; - } - else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 4) { + } else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 4) { HalfPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += HalfPacketSize; - } - else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 4) { + } else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 4) { QuarterPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += QuarterPacketSize; - } - else { + } else { const LinearMapper dm0 = rhs.getLinearMapper(k, j2); blockB[count + 0] = cj(dm0(0)); blockB[count + 1] = cj(dm0(1)); @@ -1249,17 +1240,17 @@ struct gemm_pack_rhs(zmm); - EIGEN_IF_CONSTEXPR(unrollN > U2) urolls::template transpose(zmm); - EIGEN_IF_CONSTEXPR(unrollN > U1) urolls::template transpose(zmm); + EIGEN_IF_CONSTEXPR (unrollN > U2) urolls::template transpose(zmm); + EIGEN_IF_CONSTEXPR (unrollN > U1) urolls::template transpose(zmm); static_assert((remN && unrollN == U1) || !remN, "When handling N remainder set unrollN=U1"); - EIGEN_IF_CONSTEXPR(!remN) { + EIGEN_IF_CONSTEXPR (!remN) { urolls::template storeC(C_arr, LDC, zmm, remM_); - EIGEN_IF_CONSTEXPR(unrollN > U1) { + EIGEN_IF_CONSTEXPR (unrollN > U1) { constexpr int64_t unrollN_ = std::min(unrollN - U1, U1); urolls::template storeC(C_arr + U1 * LDC, LDC, zmm, remM_); } - EIGEN_IF_CONSTEXPR(unrollN > U2) { + EIGEN_IF_CONSTEXPR (unrollN > U2) { constexpr int64_t unrollN_ = std::min(unrollN - U2, U1); urolls::template storeC(C_arr + U2 * LDC, LDC, zmm, remM_); } - } - else { - EIGEN_IF_CONSTEXPR((std::is_same::value)) { + } else { + EIGEN_IF_CONSTEXPR ((std::is_same::value)) { // Note: without "if constexpr" this section of code will also be // parsed by the compiler so each of the storeC will still be instantiated. // We use enable_if in aux_storeC to set it to an empty function for @@ -187,8 +186,7 @@ EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock(C_arr, LDC, zmm, remM_); else if (remN_ == 1) urolls::template storeC<1, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); - } - else { + } else { if (remN_ == 7) urolls::template storeC<7, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); else if (remN_ == 6) @@ -244,23 +242,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC); } } @@ -273,23 +274,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel( B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<3, 4>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<3, 4>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4); } i += 4; @@ -303,23 +307,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel( B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<3, 2>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<3, 2>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2); } i += 2; @@ -334,22 +341,25 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel( B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<3, 1>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<3, 1>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1); } } @@ -360,30 +370,33 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t int64_t i = 0; for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { Scalar *A_t = &A_arr[idA(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j]; - EIGEN_IF_CONSTEXPR(isCRowMajor) B_t = &B_arr[0 * LDB + j]; + EIGEN_IF_CONSTEXPR (isCRowMajor) B_t = &B_arr[0 * LDB + j]; PacketBlock zmm; urolls::template setzero<2, EIGEN_AVX_MAX_NUM_ROW>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC); } } @@ -396,23 +409,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<2, 4>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<2, 4>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4); } i += 4; @@ -426,23 +442,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<2, 2>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<2, 2>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2); } i += 2; @@ -456,22 +475,25 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<2, 1>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<2, 1>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1); } } @@ -488,23 +510,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC); } } @@ -517,23 +542,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<1, 4>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<1, 4>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4); } i += 4; @@ -547,23 +575,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<1, 2>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<1, 2>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2); } i += 2; @@ -578,22 +609,25 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<1, 1>(&C_arr[i * LDC + j], LDC, zmm); urolls::template storeC<1, 1>(&C_arr[i * LDC + j], LDC, zmm); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1); } } @@ -612,23 +646,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 0, N - j); } } @@ -641,23 +678,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel( B_t, A_t, LDB, LDA, zmm, N - j); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); urolls::template storeC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4, N - j); } i += 4; @@ -671,23 +711,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel( B_t, A_t, LDB, LDA, zmm, N - j); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); urolls::template storeC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2, N - j); } i += 2; @@ -701,23 +744,26 @@ void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t urolls::template microKernel( B_t, A_t, LDB, LDA, zmm, N - j); B_t += EIGEN_AVX_MAX_K_UNROL * LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; - else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t += EIGEN_AVX_MAX_K_UNROL; + else + A_t += EIGEN_AVX_MAX_K_UNROL * LDA; } - EIGEN_IF_CONSTEXPR(handleKRem) { + EIGEN_IF_CONSTEXPR (handleKRem) { for (int64_t k = K_; k < K; k++) { urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); B_t += LDB; - EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; - else A_t += LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + A_t++; + else + A_t += LDA; } } - EIGEN_IF_CONSTEXPR(isCRowMajor) { + EIGEN_IF_CONSTEXPR (isCRowMajor) { urolls::template updateC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); urolls::template storeC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); - } - else { + } else { transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1, N - j); } } @@ -840,7 +886,7 @@ EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar* B_arr, int64_t LDB, int64_t K, B_temp += U1; k += U1; } - EIGEN_IF_CONSTEXPR(U1 > 8) { + EIGEN_IF_CONSTEXPR (U1 > 8) { // Note: without "if constexpr" this section of code will also be // parsed by the compiler so there is an additional check in {load/store}BBlock // to make sure the counter is not non-negative. @@ -850,7 +896,7 @@ EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar* B_arr, int64_t LDB, int64_t K, k += 8; } } - EIGEN_IF_CONSTEXPR(U1 > 4) { + EIGEN_IF_CONSTEXPR (U1 > 4) { // Note: without "if constexpr" this section of code will also be // parsed by the compiler so there is an additional check in {load/store}BBlock // to make sure the counter is not non-negative. @@ -921,7 +967,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L int64_t sizeBTemp = 0; Scalar* B_temp = NULL; - EIGEN_IF_CONSTEXPR(!isBRowMajor) { + EIGEN_IF_CONSTEXPR (!isBRowMajor) { /** * If B is col-major, we copy it to a fixed-size temporary array of size at most ~numM*kB and * transpose it to row-major. Call the solve routine, and copy+transpose it back to the original array. @@ -930,7 +976,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L sizeBTemp = (((std::min(kB, numRHS) + psize - 1) / psize + 4) * psize) * numM; } - EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar*)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64); + EIGEN_IF_CONSTEXPR (!isBRowMajor) B_temp = (Scalar*)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64); for (int64_t k = 0; k < numRHS; k += kB) { int64_t bK = numRHS - k > kB ? kB : numRHS - k; @@ -944,7 +990,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L int64_t LDT = ((bkL + (numScalarPerCache - 1)) / numScalarPerCache) * numScalarPerCache; int64_t offsetBTemp = 0; for (int64_t i = 0; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { - EIGEN_IF_CONSTEXPR(!isBRowMajor) { + EIGEN_IF_CONSTEXPR (!isBRowMajor) { int64_t indA_i = isFWDSolve ? i : M - 1 - i; int64_t indB_i = isFWDSolve ? i : M - (i + EIGEN_AVX_MAX_NUM_ROW); int64_t offB_1 = isFWDSolve ? offsetBTemp : sizeBTemp - EIGEN_AVX_MAX_NUM_ROW * LDT - offsetBTemp; @@ -958,8 +1004,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp + offB_1, LDT); offsetBTemp += EIGEN_AVX_MAX_NUM_ROW * LDT; - } - else { + } else { int64_t ind = isFWDSolve ? i : M - 1 - i; triSolveKernelLxK( &A_arr[idA(ind, ind, LDA)], B_arr + k + ind * LDB, EIGEN_AVX_MAX_NUM_ROW, bK, LDA, LDB); @@ -977,7 +1022,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L * |__|__|__|__ |__| * |********|__| |**| */ - EIGEN_IF_CONSTEXPR(isBRowMajor) { + EIGEN_IF_CONSTEXPR (isBRowMajor) { int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW); int64_t indA_j = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW); int64_t indB_i = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW); @@ -985,8 +1030,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L gemmKernel( &A_arr[idA(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB, EIGEN_AVX_MAX_NUM_ROW, bK, i + EIGEN_AVX_MAX_NUM_ROW, LDA, LDB, LDB); - } - else { + } else { if (offsetBTemp + EIGEN_AVX_MAX_NUM_ROW * LDT > sizeBTemp) { /** * Similar idea as mentioned above, but here we are limited by the number of updated values of B @@ -1024,7 +1068,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L int64_t bM = M - M_; if (bM > 0) { if (M_ > 0) { - EIGEN_IF_CONSTEXPR(isBRowMajor) { + EIGEN_IF_CONSTEXPR (isBRowMajor) { int64_t indA_i = isFWDSolve ? M_ : 0; int64_t indA_j = isFWDSolve ? 0 : bM; int64_t indB_i = isFWDSolve ? 0 : bM; @@ -1032,8 +1076,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L gemmKernel( &A_arr[idA(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB, bM, bK, M_, LDA, LDB, LDB); - } - else { + } else { int64_t indA_i = isFWDSolve ? M_ : 0; int64_t indA_j = isFWDSolve ? gemmOff : bM; int64_t indB_i = isFWDSolve ? M_ : 0; @@ -1043,7 +1086,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L M_ - gemmOff, LDA, LDT, LDB); } } - EIGEN_IF_CONSTEXPR(!isBRowMajor) { + EIGEN_IF_CONSTEXPR (!isBRowMajor) { int64_t indA_i = isFWDSolve ? M_ : M - 1 - M_; int64_t indB_i = isFWDSolve ? M_ : 0; int64_t offB_1 = isFWDSolve ? 0 : (bM - 1) * bkL; @@ -1051,8 +1094,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L triSolveKernelLxK(&A_arr[idA(indA_i, indA_i, LDA)], B_temp + offB_1, bM, bkL, LDA, bkL); copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp, bkL, bM); - } - else { + } else { int64_t ind = isFWDSolve ? M_ : M - 1 - M_; triSolveKernelLxK(&A_arr[idA(ind, ind, LDA)], B_arr + k + ind * LDB, bM, bK, LDA, LDB); @@ -1060,7 +1102,7 @@ void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t L } } - EIGEN_IF_CONSTEXPR(!isBRowMajor) handmade_aligned_free(B_temp); + EIGEN_IF_CONSTEXPR (!isBRowMajor) handmade_aligned_free(B_temp); } // Template specializations of trsmKernelL/R for float/double and inner strides of 1. diff --git a/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc b/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc index 4538ce902..ce9eba87a 100644 --- a/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc +++ b/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc @@ -13,8 +13,10 @@ template EIGEN_ALWAYS_INLINE int64_t idA(int64_t i, int64_t j, int64_t LDA) { - EIGEN_IF_CONSTEXPR(isARowMajor) return i * LDA + j; - else return i + j * LDA; + EIGEN_IF_CONSTEXPR (isARowMajor) + return i * LDA + j; + else + return i + j * LDA; } /** @@ -59,11 +61,11 @@ namespace unrolls { template EIGEN_ALWAYS_INLINE auto remMask(int64_t m) { - EIGEN_IF_CONSTEXPR(N == 16) { return 0xFFFF >> (16 - m); } - else EIGEN_IF_CONSTEXPR(N == 8) { + EIGEN_IF_CONSTEXPR (N == 16) { + return 0xFFFF >> (16 - m); + } else EIGEN_IF_CONSTEXPR (N == 8) { return 0xFF >> (8 - m); - } - else EIGEN_IF_CONSTEXPR(N == 4) { + } else EIGEN_IF_CONSTEXPR (N == 4) { return 0x0F >> (4 - m); } return 0; @@ -155,22 +157,20 @@ class trans { constexpr int64_t counterReverse = endN - counter; constexpr int64_t startN = counterReverse; - EIGEN_IF_CONSTEXPR(startN < EIGEN_AVX_MAX_NUM_ROW) { - EIGEN_IF_CONSTEXPR(remM) { + EIGEN_IF_CONSTEXPR (startN < EIGEN_AVX_MAX_NUM_ROW) { + EIGEN_IF_CONSTEXPR (remM) { pstoreu( C_arr + LDC * startN, padd(ploadu((const Scalar *)C_arr + LDC * startN, remMask(remM_)), preinterpret(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN]), remMask(remM_)), remMask(remM_)); - } - else { + } else { pstoreu(C_arr + LDC * startN, padd(ploadu((const Scalar *)C_arr + LDC * startN), preinterpret(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN]))); } - } - else { // This block is only needed for fp32 case + } else { // This block is only needed for fp32 case // Reinterpret as __m512 for _mm512_shuffle_f32x4 vecFullFloat zmm2vecFullFloat = preinterpret( zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)]); @@ -178,15 +178,14 @@ class trans { zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)] = preinterpret(_mm512_shuffle_f32x4(zmm2vecFullFloat, zmm2vecFullFloat, 0b01001110)); - EIGEN_IF_CONSTEXPR(remM) { + EIGEN_IF_CONSTEXPR (remM) { pstoreu( C_arr + LDC * startN, padd(ploadu((const Scalar *)C_arr + LDC * startN, remMask(remM_)), preinterpret( zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)])), remMask(remM_)); - } - else { + } else { pstoreu( C_arr + LDC * startN, padd(ploadu((const Scalar *)C_arr + LDC * startN), @@ -308,16 +307,15 @@ class transB { constexpr int64_t counterReverse = endN - counter; constexpr int64_t startN = counterReverse; - EIGEN_IF_CONSTEXPR(remM) { + EIGEN_IF_CONSTEXPR (remM) { ymm.packet[packetIndexOffset + startN] = ploadu((const Scalar *)&B_arr[startN * LDB], remMask(remM_)); - } - else { - EIGEN_IF_CONSTEXPR(remN_ == 0) { + } else { + EIGEN_IF_CONSTEXPR (remN_ == 0) { ymm.packet[packetIndexOffset + startN] = ploadu((const Scalar *)&B_arr[startN * LDB]); - } - else ymm.packet[packetIndexOffset + startN] = - ploadu((const Scalar *)&B_arr[startN * LDB], remMask(remN_)); + } else + ymm.packet[packetIndexOffset + startN] = + ploadu((const Scalar *)&B_arr[startN * LDB], remMask(remN_)); } aux_loadB(B_arr, LDB, ymm, remM_); @@ -345,11 +343,10 @@ class transB { constexpr int64_t counterReverse = endN - counter; constexpr int64_t startN = counterReverse; - EIGEN_IF_CONSTEXPR(remK || remM) { + EIGEN_IF_CONSTEXPR (remK || remM) { pstoreu(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN], remMask(rem_)); - } - else { + } else { pstoreu(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN]); } @@ -406,10 +403,9 @@ class transB { constexpr int64_t counterReverse = endN - counter; constexpr int64_t startN = counterReverse; - EIGEN_IF_CONSTEXPR(toTemp) { + EIGEN_IF_CONSTEXPR (toTemp) { transB::template storeB(&B_temp[startN], LDB_, ymm, remK_); - } - else { + } else { transB::template storeB(&B_arr[0 + startN * LDB], LDB, ymm, remM_); } @@ -450,8 +446,9 @@ class transB { static EIGEN_ALWAYS_INLINE void loadBBlock(Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, PacketBlock &ymm, int64_t remM_ = 0) { - EIGEN_IF_CONSTEXPR(toTemp) { transB::template loadB(&B_arr[0], LDB, ymm, remM_); } - else { + EIGEN_IF_CONSTEXPR (toTemp) { + transB::template loadB(&B_arr[0], LDB, ymm, remM_); + } else { aux_loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); } } @@ -501,7 +498,7 @@ class transB { * * For fp32 L and U1 are 1:2 so for U3/U2 cases the loads/stores need to be split up. */ - EIGEN_IF_CONSTEXPR(unrollN == U3) { + EIGEN_IF_CONSTEXPR (unrollN == U3) { // load LxU3 B col major, transpose LxU3 row major constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U3); transB::template loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); @@ -510,7 +507,7 @@ class transB { transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm); transB::template storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); - EIGEN_IF_CONSTEXPR(maxUBlock < U3) { + EIGEN_IF_CONSTEXPR (maxUBlock < U3) { transB::template loadBBlock(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_, ymm, remM_); transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm); @@ -519,50 +516,46 @@ class transB { transB::template storeBBlock(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_, ymm, remM_); } - } - else EIGEN_IF_CONSTEXPR(unrollN == U2) { + } else EIGEN_IF_CONSTEXPR (unrollN == U2) { // load LxU2 B col major, transpose LxU2 row major constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U2); transB::template loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm); transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); - EIGEN_IF_CONSTEXPR(maxUBlock < U2) transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + EIGEN_IF_CONSTEXPR (maxUBlock < U2) transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm); transB::template storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); - EIGEN_IF_CONSTEXPR(maxUBlock < U2) { + EIGEN_IF_CONSTEXPR (maxUBlock < U2) { transB::template loadBBlock(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_, ymm, remM_); transB::template transposeLxL<0>(ymm); transB::template storeBBlock(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_, ymm, remM_); } - } - else EIGEN_IF_CONSTEXPR(unrollN == U1) { + } else EIGEN_IF_CONSTEXPR (unrollN == U1) { // load LxU1 B col major, transpose LxU1 row major transB::template loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); transB::template transposeLxL<0>(ymm); - EIGEN_IF_CONSTEXPR(EIGEN_AVX_MAX_NUM_ROW < U1) { transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); } + EIGEN_IF_CONSTEXPR (EIGEN_AVX_MAX_NUM_ROW < U1) { + transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + } transB::template storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); - } - else EIGEN_IF_CONSTEXPR(unrollN == 8 && U1 > 8) { + } else EIGEN_IF_CONSTEXPR (unrollN == 8 && U1 > 8) { // load Lx4 B col major, transpose Lx4 row major transB::template loadBBlock<8, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_); transB::template transposeLxL<0>(ymm); transB::template storeBBlock<8, toTemp, remM, 8>(B_arr, LDB, B_temp, LDB_, ymm, remM_); - } - else EIGEN_IF_CONSTEXPR(unrollN == 4 && U1 > 4) { + } else EIGEN_IF_CONSTEXPR (unrollN == 4 && U1 > 4) { // load Lx4 B col major, transpose Lx4 row major transB::template loadBBlock<4, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_); transB::template transposeLxL<0>(ymm); transB::template storeBBlock<4, toTemp, remM, 4>(B_arr, LDB, B_temp, LDB_, ymm, remM_); - } - else EIGEN_IF_CONSTEXPR(unrollN == 2) { + } else EIGEN_IF_CONSTEXPR (unrollN == 2) { // load Lx2 B col major, transpose Lx2 row major transB::template loadBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_); transB::template transposeLxL<0>(ymm); transB::template storeBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_); - } - else EIGEN_IF_CONSTEXPR(unrollN == 1) { + } else EIGEN_IF_CONSTEXPR (unrollN == 1) { // load Lx1 B col major, transpose Lx1 row major transB::template loadBBlock<1, toTemp, remM, 1>(B_arr, LDB, B_temp, LDB_, ymm, remM_); transB::template transposeLxL<0>(ymm); @@ -614,10 +607,9 @@ class trsm { constexpr int64_t packetIndex = startM * endK + startK; constexpr int64_t startM_ = isFWDSolve ? startM : -startM; const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB; - EIGEN_IF_CONSTEXPR(krem) { + EIGEN_IF_CONSTEXPR (krem) { RHSInPacket.packet[packetIndex] = ploadu(&B_arr[rhsIndex], remMask(rem)); - } - else { + } else { RHSInPacket.packet[packetIndex] = ploadu(&B_arr[rhsIndex]); } aux_loadRHS(B_arr, LDB, RHSInPacket, rem); @@ -649,10 +641,9 @@ class trsm { constexpr int64_t packetIndex = startM * endK + startK; constexpr int64_t startM_ = isFWDSolve ? startM : -startM; const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB; - EIGEN_IF_CONSTEXPR(krem) { + EIGEN_IF_CONSTEXPR (krem) { pstoreu(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex], remMask(rem)); - } - else { + } else { pstoreu(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex]); } aux_storeRHS(B_arr, LDB, RHSInPacket, rem); @@ -711,26 +702,27 @@ class trsm { // For each row of A, first update all corresponding RHS constexpr int64_t packetIndex = startM * endK + startK; - EIGEN_IF_CONSTEXPR(currentM > 0) { + EIGEN_IF_CONSTEXPR (currentM > 0) { RHSInPacket.packet[packetIndex] = pnmadd(AInPacket.packet[startM], RHSInPacket.packet[(currentM - 1) * endK + startK], RHSInPacket.packet[packetIndex]); } - EIGEN_IF_CONSTEXPR(startK == endK - 1) { + EIGEN_IF_CONSTEXPR (startK == endK - 1) { // Once all RHS for previous row of A is updated, we broadcast the next element in the column A_{i, currentM}. - EIGEN_IF_CONSTEXPR(startM == currentM && !isUnitDiag) { + EIGEN_IF_CONSTEXPR (startM == currentM && !isUnitDiag) { // If diagonal is not unit, we broadcast reciprocals of diagonals AinPacket.packet[currentM]. // This will be used in divRHSByDiag - EIGEN_IF_CONSTEXPR(isFWDSolve) - AInPacket.packet[currentM] = pset1(Scalar(1) / A_arr[idA(currentM, currentM, LDA)]); - else AInPacket.packet[currentM] = pset1(Scalar(1) / A_arr[idA(-currentM, -currentM, LDA)]); - } - else { + EIGEN_IF_CONSTEXPR (isFWDSolve) + AInPacket.packet[currentM] = pset1(Scalar(1) / A_arr[idA(currentM, currentM, LDA)]); + else + AInPacket.packet[currentM] = pset1(Scalar(1) / A_arr[idA(-currentM, -currentM, LDA)]); + } else { // Broadcast next off diagonal element of A - EIGEN_IF_CONSTEXPR(isFWDSolve) - AInPacket.packet[startM] = pset1(A_arr[idA(startM, currentM, LDA)]); - else AInPacket.packet[startM] = pset1(A_arr[idA(-startM, -currentM, LDA)]); + EIGEN_IF_CONSTEXPR (isFWDSolve) + AInPacket.packet[startM] = pset1(A_arr[idA(startM, currentM, LDA)]); + else + AInPacket.packet[startM] = pset1(A_arr[idA(-startM, -currentM, LDA)]); } } @@ -769,8 +761,8 @@ class trsm { // Without "if constexpr" the compiler instantiates the case <-1, numK> // this is handled with enable_if to prevent out-of-bound warnings // from the compiler - EIGEN_IF_CONSTEXPR(!isUnitDiag && startM > 0) - trsm::template divRHSByDiag(RHSInPacket, AInPacket); + EIGEN_IF_CONSTEXPR (!isUnitDiag && startM > 0) + trsm::template divRHSByDiag(RHSInPacket, AInPacket); // After division, the rhs corresponding to subsequent rows of A can be partially updated // We also broadcast the reciprocal of the next diagonal to AInPacket.packet[currentM] (if needed) @@ -779,8 +771,8 @@ class trsm { AInPacket); // Handle division for the RHS corresponding to the final row of A. - EIGEN_IF_CONSTEXPR(!isUnitDiag && startM == endM - 1) - trsm::template divRHSByDiag(RHSInPacket, AInPacket); + EIGEN_IF_CONSTEXPR (!isUnitDiag && startM == endM - 1) + trsm::template divRHSByDiag(RHSInPacket, AInPacket); aux_triSolveMicroKernel(A_arr, LDA, RHSInPacket, AInPacket); @@ -916,12 +908,13 @@ class gemm { constexpr int64_t startM = counterReverse / (endN); constexpr int64_t startN = counterReverse % endN; - EIGEN_IF_CONSTEXPR(rem) - zmm.packet[startN * endM + startM] = - padd(ploadu(&C_arr[(startN)*LDC + startM * PacketSize], remMask(rem_)), - zmm.packet[startN * endM + startM], remMask(rem_)); - else zmm.packet[startN * endM + startM] = - padd(ploadu(&C_arr[(startN)*LDC + startM * PacketSize]), zmm.packet[startN * endM + startM]); + EIGEN_IF_CONSTEXPR (rem) + zmm.packet[startN * endM + startM] = + padd(ploadu(&C_arr[(startN)*LDC + startM * PacketSize], remMask(rem_)), + zmm.packet[startN * endM + startM], remMask(rem_)); + else + zmm.packet[startN * endM + startM] = + padd(ploadu(&C_arr[(startN)*LDC + startM * PacketSize]), zmm.packet[startN * endM + startM]); aux_updateC(C_arr, LDC, zmm, rem_); } @@ -949,10 +942,11 @@ class gemm { constexpr int64_t startM = counterReverse / (endN); constexpr int64_t startN = counterReverse % endN; - EIGEN_IF_CONSTEXPR(rem) - pstoreu(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM], - remMask(rem_)); - else pstoreu(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM]); + EIGEN_IF_CONSTEXPR (rem) + pstoreu(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM], + remMask(rem_)); + else + pstoreu(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM]); aux_storeC(C_arr, LDC, zmm, rem_); } @@ -978,11 +972,12 @@ class gemm { constexpr int64_t counterReverse = endL - counter; constexpr int64_t startL = counterReverse; - EIGEN_IF_CONSTEXPR(rem) - zmm.packet[unrollM * unrollN + startL] = - ploadu(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize], remMask(rem_)); - else zmm.packet[unrollM * unrollN + startL] = - ploadu(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize]); + EIGEN_IF_CONSTEXPR (rem) + zmm.packet[unrollM * unrollN + startL] = + ploadu(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize], remMask(rem_)); + else + zmm.packet[unrollM * unrollN + startL] = + ploadu(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize]); aux_startLoadB(B_t, LDB, zmm, rem_); } @@ -1037,11 +1032,10 @@ class gemm { constexpr int64_t counterReverse = endM - counter; constexpr int64_t startM = counterReverse; - EIGEN_IF_CONSTEXPR(rem) { + EIGEN_IF_CONSTEXPR (rem) { zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] = ploadu(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize], remMask(rem_)); - } - else { + } else { zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] = ploadu(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize]); } @@ -1079,32 +1073,31 @@ class gemm { constexpr int startN = (counterReverse / (endM)) % endN; constexpr int startM = counterReverse % endM; - EIGEN_IF_CONSTEXPR(startK == 0 && startM == 0 && startN == 0) { + EIGEN_IF_CONSTEXPR (startK == 0 && startM == 0 && startN == 0) { gemm::template startLoadB(B_t, LDB, zmm, rem_); gemm::template startBCastA(A_t, LDA, zmm); } { // Interleave FMA and Bcast - EIGEN_IF_CONSTEXPR(isAdd) { + EIGEN_IF_CONSTEXPR (isAdd) { zmm.packet[startN * endM + startM] = pmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast], zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]); - } - else { + } else { zmm.packet[startN * endM + startM] = pnmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast], zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]); } // Bcast - EIGEN_IF_CONSTEXPR(startM == endM - 1 && (numBCast + startN + startK * endN < endK * endN)) { + EIGEN_IF_CONSTEXPR (startM == endM - 1 && (numBCast + startN + startK * endN < endK * endN)) { zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast] = pload1(&A_t[idA( (numBCast + startN + startK * endN) % endN, (numBCast + startN + startK * endN) / endN, LDA)]); } } // We have updated all accumulators, time to load next set of B's - EIGEN_IF_CONSTEXPR((startN == endN - 1) && (startM == endM - 1)) { + EIGEN_IF_CONSTEXPR ((startN == endN - 1) && (startM == endM - 1)) { gemm::template loadB(B_t, LDB, zmm, rem_); } aux_microKernel(B_t, A_t, LDB, LDA, zmm, rem_); diff --git a/Eigen/src/Core/arch/AltiVec/MatrixProduct.h b/Eigen/src/Core/arch/AltiVec/MatrixProduct.h index 2dfd6baa4..a2d3148bf 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixProduct.h +++ b/Eigen/src/Core/arch/AltiVec/MatrixProduct.h @@ -368,8 +368,12 @@ EIGEN_ALWAYS_INLINE void storeBlock(Scalar* to, PacketBlock& block) { const Index size = 16 / sizeof(Scalar); pstore(to + (0 * size), block.packet[0]); pstore(to + (1 * size), block.packet[1]); - EIGEN_IF_CONSTEXPR(N > 2) { pstore(to + (2 * size), block.packet[2]); } - EIGEN_IF_CONSTEXPR(N > 3) { pstore(to + (3 * size), block.packet[3]); } + EIGEN_IF_CONSTEXPR (N > 2) { + pstore(to + (2 * size), block.packet[2]); + } + EIGEN_IF_CONSTEXPR (N > 3) { + pstore(to + (3 * size), block.packet[3]); + } } // General template for lhs & rhs complex packing. @@ -420,21 +424,21 @@ struct dhs_cpack { PacketBlock cblock; for (; i + vectorSize <= depth; i += vectorSize) { - EIGEN_IF_CONSTEXPR(UseLhs) { bload(cblock, lhs2, 0, i); } - else { + EIGEN_IF_CONSTEXPR (UseLhs) { + bload(cblock, lhs2, 0, i); + } else { bload(cblock, lhs2, i, 0); } - EIGEN_IF_CONSTEXPR(((StorageOrder == RowMajor) && UseLhs) || (((StorageOrder == ColMajor) && !UseLhs))) { + EIGEN_IF_CONSTEXPR (((StorageOrder == RowMajor) && UseLhs) || (((StorageOrder == ColMajor) && !UseLhs))) { dhs_cblock(cblock, blockr, p16uc_GETREAL32b); dhs_cblock(cblock, blocki, p16uc_GETIMAG32b); - } - else { + } else { dhs_cblock(cblock, blockr, p16uc_GETREAL32); dhs_cblock(cblock, blocki, p16uc_GETIMAG32); } - EIGEN_IF_CONSTEXPR(Conjugate) { + EIGEN_IF_CONSTEXPR (Conjugate) { blocki.packet[0] = -blocki.packet[0]; blocki.packet[1] = -blocki.packet[1]; blocki.packet[2] = -blocki.packet[2]; @@ -469,22 +473,19 @@ struct dhs_cpack { PacketBlock blockr, blocki; PacketBlock cblock; - EIGEN_IF_CONSTEXPR(((StorageOrder == ColMajor) && UseLhs) || (((StorageOrder == RowMajor) && !UseLhs))) { - EIGEN_IF_CONSTEXPR(UseLhs) { + EIGEN_IF_CONSTEXPR (((StorageOrder == ColMajor) && UseLhs) || (((StorageOrder == RowMajor) && !UseLhs))) { + EIGEN_IF_CONSTEXPR (UseLhs) { cblock.packet[0] = lhs2.template loadPacket(0, i); cblock.packet[1] = lhs2.template loadPacket(2, i); - } - else { + } else { cblock.packet[0] = lhs2.template loadPacket(i, 0); cblock.packet[1] = lhs2.template loadPacket(i, 2); } - } - else { - EIGEN_IF_CONSTEXPR(UseLhs) { + } else { + EIGEN_IF_CONSTEXPR (UseLhs) { cblock.packet[0] = pload2(lhs2(0, i), lhs2(1, i)); cblock.packet[1] = pload2(lhs2(2, i), lhs2(3, i)); - } - else { + } else { cblock.packet[0] = pload2(lhs2(i, 0), lhs2(i, 1)); cblock.packet[1] = pload2(lhs2(i, 2), lhs2(i, 3)); } @@ -493,7 +494,9 @@ struct dhs_cpack { blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL32); blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG32); - EIGEN_IF_CONSTEXPR(Conjugate) { blocki.packet[0] = -blocki.packet[0]; } + EIGEN_IF_CONSTEXPR (Conjugate) { + blocki.packet[0] = -blocki.packet[0]; + } pstore(blockAt + rir, blockr.packet[0]); pstore(blockAt + rii, blocki.packet[0]); @@ -505,8 +508,8 @@ struct dhs_cpack { rir += ((PanelMode) ? (vectorSize * (2 * stride - depth)) : vectorDelta); } - EIGEN_IF_CONSTEXPR(!UseLhs) { - EIGEN_IF_CONSTEXPR(PanelMode) rir -= (offset * (vectorSize - 1)); + EIGEN_IF_CONSTEXPR (!UseLhs) { + EIGEN_IF_CONSTEXPR (PanelMode) rir -= (offset * (vectorSize - 1)); for (; j < rows; j++) { const DataMapper lhs2 = lhs.getSubMapper(0, j); @@ -515,9 +518,10 @@ struct dhs_cpack { for (Index i = 0; i < depth; i++) { blockAt[rir] = lhs2(i, 0).real(); - EIGEN_IF_CONSTEXPR(Conjugate) - blockAt[rii] = -lhs2(i, 0).imag(); - else blockAt[rii] = lhs2(i, 0).imag(); + EIGEN_IF_CONSTEXPR (Conjugate) + blockAt[rii] = -lhs2(i, 0).imag(); + else + blockAt[rii] = lhs2(i, 0).imag(); rir += 1; rii += 1; @@ -525,10 +529,9 @@ struct dhs_cpack { rir += ((PanelMode) ? (2 * stride - depth) : depth); } - } - else { + } else { if (j < rows) { - EIGEN_IF_CONSTEXPR(PanelMode) rir += (offset * (rows - j - vectorSize)); + EIGEN_IF_CONSTEXPR (PanelMode) rir += (offset * (rows - j - vectorSize)); rii = rir + (((PanelMode) ? stride : depth) * (rows - j)); for (Index i = 0; i < depth; i++) { @@ -536,9 +539,10 @@ struct dhs_cpack { for (; k < rows; k++) { blockAt[rir] = lhs(k, i).real(); - EIGEN_IF_CONSTEXPR(Conjugate) - blockAt[rii] = -lhs(k, i).imag(); - else blockAt[rii] = lhs(k, i).imag(); + EIGEN_IF_CONSTEXPR (Conjugate) + blockAt[rii] = -lhs(k, i).imag(); + else + blockAt[rii] = lhs(k, i).imag(); rir += 1; rii += 1; @@ -559,15 +563,14 @@ struct dhs_pack { for (; i + n * vectorSize <= depth; i += n * vectorSize) { for (Index k = 0; k < n; k++) { - EIGEN_IF_CONSTEXPR(UseLhs) { + EIGEN_IF_CONSTEXPR (UseLhs) { bload(block[k], lhs2, 0, i + k * vectorSize); - } - else { + } else { bload(block[k], lhs2, i + k * vectorSize, 0); } } - EIGEN_IF_CONSTEXPR(((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { + EIGEN_IF_CONSTEXPR (((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { for (Index k = 0; k < n; k++) { ptranspose(block[k]); } @@ -590,31 +593,30 @@ struct dhs_pack { const DataMapper lhs2 = UseLhs ? lhs.getSubMapper(j, 0) : lhs.getSubMapper(0, j); Index i = 0; - EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * offset; + EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * offset; dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize); dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize); dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize); for (; i < depth; i++) { - EIGEN_IF_CONSTEXPR(((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { - EIGEN_IF_CONSTEXPR(UseLhs) { + EIGEN_IF_CONSTEXPR (((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) { + EIGEN_IF_CONSTEXPR (UseLhs) { blockA[ri + 0] = lhs2(0, i); blockA[ri + 1] = lhs2(1, i); blockA[ri + 2] = lhs2(2, i); blockA[ri + 3] = lhs2(3, i); - } - else { + } else { blockA[ri + 0] = lhs2(i, 0); blockA[ri + 1] = lhs2(i, 1); blockA[ri + 2] = lhs2(i, 2); blockA[ri + 3] = lhs2(i, 3); } - } - else { + } else { Packet lhsV; - EIGEN_IF_CONSTEXPR(UseLhs) { lhsV = lhs2.template loadPacket(0, i); } - else { + EIGEN_IF_CONSTEXPR (UseLhs) { + lhsV = lhs2.template loadPacket(0, i); + } else { lhsV = lhs2.template loadPacket(i, 0); } pstore(blockA + ri, lhsV); @@ -623,11 +625,11 @@ struct dhs_pack { ri += vectorSize; } - EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * (stride - offset - depth); + EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * (stride - offset - depth); } - EIGEN_IF_CONSTEXPR(!UseLhs) { - EIGEN_IF_CONSTEXPR(PanelMode) ri += offset; + EIGEN_IF_CONSTEXPR (!UseLhs) { + EIGEN_IF_CONSTEXPR (PanelMode) ri += offset; for (; j < rows; j++) { const DataMapper lhs2 = lhs.getSubMapper(0, j); @@ -636,12 +638,11 @@ struct dhs_pack { ri += 1; } - EIGEN_IF_CONSTEXPR(PanelMode) ri += stride - depth; + EIGEN_IF_CONSTEXPR (PanelMode) ri += stride - depth; } - } - else { + } else { if (j < rows) { - EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (rows - j); + EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (rows - j); for (Index i = 0; i < depth; i++) { Index k = j; @@ -665,17 +666,16 @@ struct dhs_pack { for (; i + n * vectorSize <= depth; i += n * vectorSize) { for (Index k = 0; k < n; k++) { - EIGEN_IF_CONSTEXPR(StorageOrder == RowMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == RowMajor) { block[k].packet[0] = lhs2.template loadPacket(0, i + k * vectorSize); block[k].packet[1] = lhs2.template loadPacket(1, i + k * vectorSize); - } - else { + } else { block[k].packet[0] = lhs2.template loadPacket(0, i + k * vectorSize + 0); block[k].packet[1] = lhs2.template loadPacket(0, i + k * vectorSize + 1); } } - EIGEN_IF_CONSTEXPR(StorageOrder == RowMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == RowMajor) { for (Index k = 0; k < n; k++) { ptranspose(block[k]); } @@ -698,18 +698,17 @@ struct dhs_pack { const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * offset; + EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * offset; dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize); dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize); dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize); for (; i < depth; i++) { - EIGEN_IF_CONSTEXPR(StorageOrder == RowMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == RowMajor) { blockA[ri + 0] = lhs2(0, i); blockA[ri + 1] = lhs2(1, i); - } - else { + } else { Packet2d lhsV = lhs2.template loadPacket(0, i); pstore(blockA + ri, lhsV); } @@ -717,11 +716,11 @@ struct dhs_pack { ri += vectorSize; } - EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * (stride - offset - depth); + EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * (stride - offset - depth); } if (j < rows) { - EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (rows - j); + EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (rows - j); for (Index i = 0; i < depth; i++) { Index k = j; @@ -745,13 +744,12 @@ struct dhs_pack { for (; i + n * vectorSize <= depth; i += n * vectorSize) { for (Index k = 0; k < n; k++) { - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { block1[k].packet[0] = rhs2.template loadPacket(i + k * vectorSize, 0); block1[k].packet[1] = rhs2.template loadPacket(i + k * vectorSize, 1); block2[k].packet[0] = rhs2.template loadPacket(i + k * vectorSize, 2); block2[k].packet[1] = rhs2.template loadPacket(i + k * vectorSize, 3); - } - else { + } else { block3[k].packet[0] = rhs2.template loadPacket(i + k * vectorSize + 0, 0); //[a1 a2] block3[k].packet[1] = rhs2.template loadPacket(i + k * vectorSize + 0, 2); //[a3 a4] block3[k].packet[2] = rhs2.template loadPacket(i + k * vectorSize + 1, 0); //[b1 b2] @@ -759,7 +757,7 @@ struct dhs_pack { } } - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { for (Index k = 0; k < n; k++) { ptranspose(block1[k]); ptranspose(block2[k]); @@ -767,13 +765,12 @@ struct dhs_pack { } for (Index k = 0; k < n; k++) { - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { pstore(blockB + ri + k * 4 * vectorSize, block1[k].packet[0]); pstore(blockB + ri + k * 4 * vectorSize + 2, block2[k].packet[0]); pstore(blockB + ri + k * 4 * vectorSize + 4, block1[k].packet[1]); pstore(blockB + ri + k * 4 * vectorSize + 6, block2[k].packet[1]); - } - else { + } else { storeBlock(blockB + ri + k * 4 * vectorSize, block3[k]); } } @@ -791,14 +788,14 @@ struct dhs_pack { const DataMapper rhs2 = rhs.getSubMapper(0, j); Index i = 0; - EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (2 * vectorSize); + EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (2 * vectorSize); dhs_copy<4>(blockB, rhs2, i, ri, depth, vectorSize); dhs_copy<2>(blockB, rhs2, i, ri, depth, vectorSize); dhs_copy<1>(blockB, rhs2, i, ri, depth, vectorSize); for (; i < depth; i++) { - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { blockB[ri + 0] = rhs2(i, 0); blockB[ri + 1] = rhs2(i, 1); @@ -806,8 +803,7 @@ struct dhs_pack { blockB[ri + 0] = rhs2(i, 2); blockB[ri + 1] = rhs2(i, 3); - } - else { + } else { Packet2d rhsV = rhs2.template loadPacket(i, 0); pstore(blockB + ri, rhsV); @@ -819,10 +815,10 @@ struct dhs_pack { ri += vectorSize; } - EIGEN_IF_CONSTEXPR(PanelMode) ri += (2 * vectorSize) * (stride - offset - depth); + EIGEN_IF_CONSTEXPR (PanelMode) ri += (2 * vectorSize) * (stride - offset - depth); } - EIGEN_IF_CONSTEXPR(PanelMode) ri += offset; + EIGEN_IF_CONSTEXPR (PanelMode) ri += offset; for (; j < cols; j++) { const DataMapper rhs2 = rhs.getSubMapper(0, j); @@ -831,7 +827,7 @@ struct dhs_pack { ri += 1; } - EIGEN_IF_CONSTEXPR(PanelMode) ri += stride - depth; + EIGEN_IF_CONSTEXPR (PanelMode) ri += stride - depth; } } }; @@ -848,9 +844,9 @@ struct dhs_pack const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - EIGEN_IF_CONSTEXPR(PanelMode) ri += 2 * vectorSize * offset; + EIGEN_IF_CONSTEXPR (PanelMode) ri += 2 * vectorSize * offset; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { for (; i + 2 <= depth; i += 2) { PacketBlock block; @@ -881,8 +877,7 @@ struct dhs_pack ri += 2 * vectorSize; } - } - else { + } else { for (; i + vectorSize <= depth; i += vectorSize) { PacketBlock block1, block2; @@ -1001,15 +996,15 @@ struct dhs_pack } } - EIGEN_IF_CONSTEXPR(PanelMode) ri += 2 * vectorSize * (stride - offset - depth); + EIGEN_IF_CONSTEXPR (PanelMode) ri += 2 * vectorSize * (stride - offset - depth); } for (; j + vectorSize <= rows; j += vectorSize) { const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * offset; + EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * offset; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { for (; i + 2 <= depth; i += 2) { PacketBlock block; @@ -1031,8 +1026,7 @@ struct dhs_pack ri += vectorSize; } - } - else { + } else { for (; i + vectorSize <= depth; i += vectorSize) { PacketBlock block1; @@ -1109,16 +1103,16 @@ struct dhs_pack } } - EIGEN_IF_CONSTEXPR(PanelMode) ri += vectorSize * (stride - offset - depth); + EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * (stride - offset - depth); } if (j + 4 <= rows) { const DataMapper lhs2 = lhs.getSubMapper(j, 0); Index i = 0; - EIGEN_IF_CONSTEXPR(PanelMode) ri += 4 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * offset; for (; i + 2 <= depth; i += 2) { - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { PacketBlock block; block.packet[0] = lhs2.template loadPacketPartial(0, i + 0, 4); @@ -1127,8 +1121,7 @@ struct dhs_pack block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val); pstore(blockA + ri, block.packet[0]); - } - else { + } else { blockA[ri + 0] = lhs2(0, i + 0); blockA[ri + 1] = lhs2(0, i + 1); blockA[ri + 2] = lhs2(1, i + 0); @@ -1142,12 +1135,11 @@ struct dhs_pack ri += 2 * 4; } if (depth & 1) { - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { Packet8bf lhsV = lhs2.template loadPacketPartial(0, i + 0, 4); pstore_partial(blockA + ri, lhsV, 4); - } - else { + } else { blockA[ri + 0] = lhs2(0, i); blockA[ri + 1] = lhs2(1, i); blockA[ri + 2] = lhs2(2, i); @@ -1157,12 +1149,12 @@ struct dhs_pack ri += 4; } - EIGEN_IF_CONSTEXPR(PanelMode) ri += 4 * (stride - offset - depth); + EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * (stride - offset - depth); j += 4; } if (j < rows) { - EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (rows - j); + EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (rows - j); Index i = 0; for (; i + 2 <= depth; i += 2) { @@ -1195,10 +1187,10 @@ struct dhs_pack const DataMapper rhs2 = rhs.getSubMapper(0, j); Index i = 0; - EIGEN_IF_CONSTEXPR(PanelMode) ri += 4 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * offset; for (; i + vectorSize <= depth; i += vectorSize) { - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { PacketBlock block; bload(block, rhs2, i, 0); @@ -1231,8 +1223,7 @@ struct dhs_pack #endif storeBlock(blockB + ri, block); - } - else { + } else { PacketBlock block; for (int M = 0; M < 8; M++) { @@ -1254,7 +1245,7 @@ struct dhs_pack ri += 4 * vectorSize; } for (; i + 2 <= depth; i += 2) { - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { blockB[ri + 0] = rhs2(i + 0, 0); blockB[ri + 1] = rhs2(i + 1, 0); blockB[ri + 2] = rhs2(i + 0, 1); @@ -1263,8 +1254,7 @@ struct dhs_pack blockB[ri + 5] = rhs2(i + 1, 2); blockB[ri + 6] = rhs2(i + 0, 3); blockB[ri + 7] = rhs2(i + 1, 3); - } - else { + } else { PacketBlock block; for (int M = 0; M < 2; M++) { @@ -1287,11 +1277,11 @@ struct dhs_pack ri += 4; } - EIGEN_IF_CONSTEXPR(PanelMode) ri += 4 * (stride - offset - depth); + EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * (stride - offset - depth); } if (j < cols) { - EIGEN_IF_CONSTEXPR(PanelMode) ri += offset * (cols - j); + EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (cols - j); Index i = 0; for (; i + 2 <= depth; i += 2) { @@ -1321,7 +1311,7 @@ struct dhs_cpack cblock; for (; i + vectorSize <= depth; i += vectorSize) { - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { cblock.packet[0] = lhs2.template loadPacket(0, i + 0); //[a1 a1i] cblock.packet[1] = lhs2.template loadPacket(0, i + 1); //[b1 b1i] @@ -1333,8 +1323,7 @@ struct dhs_cpack(0, i); //[a1 a1i] cblock.packet[1] = lhs2.template loadPacket(1, i); //[a2 a2i] @@ -1348,7 +1337,7 @@ struct dhs_cpack(blockAt + rir, blockr.packet[0]); pstore(blockAt + rii, blocki.packet[0]); @@ -1400,7 +1391,7 @@ struct dhs_cpack EIGEN_ALWAYS_INLINE void pger_common(PacketBlock* acc, const Packet& lhsV, const Packet* rhsV) { - EIGEN_IF_CONSTEXPR(NegativeAccumulate) { + EIGEN_IF_CONSTEXPR (NegativeAccumulate) { for (int M = 0; M < N; M++) { acc->packet[M] = vec_nmsub(lhsV, rhsV[M], acc->packet[M]); } - } - else { + } else { for (int M = 0; M < N; M++) { acc->packet[M] = vec_madd(lhsV, rhsV[M], acc->packet[M]); } @@ -1523,16 +1515,14 @@ template * accReal, PacketBlock* accImag, const Packet& lhsV, Packet& lhsVi, const Packet* rhsV, const Packet* rhsVi) { pger_common(accReal, lhsV, rhsV); - EIGEN_IF_CONSTEXPR(LhsIsReal) { + EIGEN_IF_CONSTEXPR (LhsIsReal) { pger_common(accImag, lhsV, rhsVi); EIGEN_UNUSED_VARIABLE(lhsVi); - } - else { - EIGEN_IF_CONSTEXPR(!RhsIsReal) { + } else { + EIGEN_IF_CONSTEXPR (!RhsIsReal) { pger_common(accReal, lhsVi, rhsVi); pger_common(accImag, lhsV, rhsVi); - } - else { + } else { EIGEN_UNUSED_VARIABLE(rhsVi); } pger_common(accImag, lhsVi, rhsV); @@ -1544,9 +1534,10 @@ EIGEN_ALWAYS_INLINE void pgerc(PacketBlock* accReal, PacketBlock(lhs_ptr); Packet lhsVi; - EIGEN_IF_CONSTEXPR(!LhsIsReal) - lhsVi = ploadLhs(lhs_ptr_imag); - else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); + EIGEN_IF_CONSTEXPR (!LhsIsReal) + lhsVi = ploadLhs(lhs_ptr_imag); + else + EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); pgerc_common(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi); } @@ -1584,11 +1575,10 @@ template EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock& aImag, const Packet& bReal, const Packet& bImag, PacketBlock& cReal, PacketBlock& cImag, const Packet& pMask) { - EIGEN_IF_CONSTEXPR(mask && (sizeof(__UNPACK_TYPE__(Packet)) == sizeof(float))) { + EIGEN_IF_CONSTEXPR (mask && (sizeof(__UNPACK_TYPE__(Packet)) == sizeof(float))) { band(aReal, pMask); band(aImag, pMask); - } - else { + } else { EIGEN_UNUSED_VARIABLE(pMask); } @@ -1607,21 +1597,20 @@ EIGEN_ALWAYS_INLINE void bscalec(PacketBlock& aReal, PacketBlock EIGEN_ALWAYS_INLINE void bload(PacketBlock& acc, const DataMapper& res, Index row, Index col) { - EIGEN_IF_CONSTEXPR(StorageOrder == RowMajor) { + EIGEN_IF_CONSTEXPR (StorageOrder == RowMajor) { for (int M = 0; M < N; M++) { acc.packet[M] = res.template loadPacket(row + M, col); } - EIGEN_IF_CONSTEXPR(Complex) { + EIGEN_IF_CONSTEXPR (Complex) { for (int M = 0; M < N; M++) { acc.packet[M + N] = res.template loadPacket(row + M, col + accCols); } } - } - else { + } else { for (int M = 0; M < N; M++) { acc.packet[M] = res.template loadPacket(row, col + M); } - EIGEN_IF_CONSTEXPR(Complex && full) { + EIGEN_IF_CONSTEXPR (Complex && full) { for (int M = 0; M < N; M++) { acc.packet[M + N] = res.template loadPacket(row + accCols, col + M); } @@ -1643,7 +1632,7 @@ EIGEN_ALWAYS_INLINE void bload_partial(PacketBlock& for (Index M = 0; M < N; M++) { acc.packet[M] = res.template loadPacketPartial(row, M, elements); } - EIGEN_IF_CONSTEXPR(Complex && full) { + EIGEN_IF_CONSTEXPR (Complex && full) { for (Index M = 0; M < N; M++) { acc.packet[M + N] = res.template loadPacketPartial(row + accCols, M, elements); } @@ -1707,8 +1696,9 @@ EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock EIGEN_ALWAYS_INLINE void bscale(PacketBlock& acc, PacketBlock& accZ, const Packet& pAlpha, const Packet& pMask) { - EIGEN_IF_CONSTEXPR(mask) { band(accZ, pMask); } - else { + EIGEN_IF_CONSTEXPR (mask) { + band(accZ, pMask); + } else { EIGEN_UNUSED_VARIABLE(pMask); } @@ -1720,21 +1710,22 @@ EIGEN_ALWAYS_INLINE void pbroadcastN(const __UNPACK_TYPE__(Packet) * ap0, const const __UNPACK_TYPE__(Packet) * ap2, Packet& a0, Packet& a1, Packet& a2, Packet& a3) { a0 = pset1(ap0[0]); - EIGEN_IF_CONSTEXPR(N == 4) { + EIGEN_IF_CONSTEXPR (N == 4) { a1 = pset1(ap0[1]); a2 = pset1(ap0[2]); a3 = pset1(ap0[3]); EIGEN_UNUSED_VARIABLE(ap1); EIGEN_UNUSED_VARIABLE(ap2); - } - else { - EIGEN_IF_CONSTEXPR(N > 1) { a1 = pset1(ap1[0]); } - else { + } else { + EIGEN_IF_CONSTEXPR (N > 1) { + a1 = pset1(ap1[0]); + } else { EIGEN_UNUSED_VARIABLE(a1); EIGEN_UNUSED_VARIABLE(ap1); } - EIGEN_IF_CONSTEXPR(N > 2) { a2 = pset1(ap2[0]); } - else { + EIGEN_IF_CONSTEXPR (N > 2) { + a2 = pset1(ap2[0]); + } else { EIGEN_UNUSED_VARIABLE(a2); EIGEN_UNUSED_VARIABLE(ap2); } @@ -1772,7 +1763,7 @@ EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock& taccReal, Packet acc1.packet[M].v = vec_mergeh(taccReal.packet[M], taccImag.packet[M]); } - EIGEN_IF_CONSTEXPR(full) { + EIGEN_IF_CONSTEXPR (full) { for (int M = 0; M < N; M++) { acc2.packet[M].v = vec_mergel(taccReal.packet[M], taccImag.packet[M]); } @@ -1789,7 +1780,7 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

(tRes.packet[M], acc1.packet[M]); } - EIGEN_IF_CONSTEXPR(full) { + EIGEN_IF_CONSTEXPR (full) { for (int M = 0; M < N; M++) { acc2.packet[M] = padd(tRes.packet[M + N], acc2.packet[M]); } @@ -1808,28 +1799,31 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

peel) && (peel != 0)) { bsetzero(accZero##peel); } \ - else { \ - EIGEN_UNUSED_VARIABLE(accZero##peel); \ +#define MICRO_ZERO_PEEL(peel) \ + EIGEN_IF_CONSTEXPR ((PEEL_ROW > peel) && (peel != 0)) { \ + bsetzero(accZero##peel); \ + } else { \ + EIGEN_UNUSED_VARIABLE(accZero##peel); \ } -#define MICRO_ADD(ptr, N) \ - EIGEN_IF_CONSTEXPR(MICRO_NORMAL_ROWS) { MICRO_RHS(ptr, 0) += (accRows * N); } \ - else { \ - MICRO_RHS(ptr, 0) += N; \ - MICRO_RHS(ptr, 1) += N; \ - EIGEN_IF_CONSTEXPR(accRows == 3) { MICRO_RHS(ptr, 2) += N; } \ +#define MICRO_ADD(ptr, N) \ + EIGEN_IF_CONSTEXPR (MICRO_NORMAL_ROWS) { \ + MICRO_RHS(ptr, 0) += (accRows * N); \ + } else { \ + MICRO_RHS(ptr, 0) += N; \ + MICRO_RHS(ptr, 1) += N; \ + EIGEN_IF_CONSTEXPR (accRows == 3) { \ + MICRO_RHS(ptr, 2) += N; \ + } \ } #define MICRO_ADD_ROWS(N) MICRO_ADD(ptr, N) #define MICRO_BROADCAST1(peel, ptr, rhsV, real) \ - EIGEN_IF_CONSTEXPR(MICRO_NORMAL_ROWS) { \ + EIGEN_IF_CONSTEXPR (MICRO_NORMAL_ROWS) { \ pbroadcastN(MICRO_RHS(ptr, 0) + (accRows * peel), MICRO_RHS(ptr, 0), MICRO_RHS(ptr, 0), \ rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ - } \ - else { \ + } else { \ pbroadcastN(MICRO_RHS(ptr, 0) + peel, MICRO_RHS(ptr, 1) + peel, MICRO_RHS(ptr, 2) + peel, \ rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ } @@ -1845,18 +1839,18 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

& taccReal, PacketBlock

peel) { \ + EIGEN_IF_CONSTEXPR (PEEL_ROW > peel) { \ MICRO_BROADCAST(peel) \ pger(&accZero##peel, lhs_ptr + (remaining_rows * peel), rhsV##peel); \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ } @@ -1879,7 +1872,7 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

peel) { \ + EIGEN_IF_CONSTEXPR (PEEL_ROW > peel) { \ for (Index i = 0; i < accRows; i++) { \ accZero##sum.packet[i] += accZero##peel.packet[i]; \ } \ @@ -1890,18 +1883,22 @@ EIGEN_ALWAYS_INLINE void bcouple(PacketBlock& taccReal, PacketBlock

EIGEN_ALWAYS_INLINE void MICRO_EXTRA_ROW(const Scalar*& lhs_ptr, const Scalar*& rhs_ptr0, const Scalar*& rhs_ptr1, @@ -1961,17 +1958,21 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_row_iteration(const DataMapper& res, cons #endif } -#define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col) \ - switch (value) { \ - default: \ - MICRO_EXTRA_UNROLL(1) \ - break; \ - case 2: \ - EIGEN_IF_CONSTEXPR(is_col || (sizeof(Scalar) == sizeof(float))) { MICRO_EXTRA_UNROLL(2) } \ - break; \ - case 3: \ - EIGEN_IF_CONSTEXPR(is_col || (sizeof(Scalar) == sizeof(float))) { MICRO_EXTRA_UNROLL(3) } \ - break; \ +#define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col) \ + switch (value) { \ + default: \ + MICRO_EXTRA_UNROLL(1) \ + break; \ + case 2: \ + EIGEN_IF_CONSTEXPR (is_col || (sizeof(Scalar) == sizeof(float))) { \ + MICRO_EXTRA_UNROLL(2) \ + } \ + break; \ + case 3: \ + EIGEN_IF_CONSTEXPR (is_col || (sizeof(Scalar) == sizeof(float))) { \ + MICRO_EXTRA_UNROLL(3) \ + } \ + break; \ } #define MICRO_EXTRA_ROWS(N) \ @@ -1990,17 +1991,16 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs func(0, peel) func(1, peel) func(2, peel) func(3, peel) func(4, peel) func(5, peel) func(6, peel) func(7, peel) #define MICRO_WORK_ONE(iter, peel) \ - EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ pger_common(&accZero##iter, lhsV##iter, rhsV##peel); \ } #define MICRO_TYPE_PEEL4(func, func2, peel) \ - EIGEN_IF_CONSTEXPR(PEEL > peel) { \ + EIGEN_IF_CONSTEXPR (PEEL > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \ MICRO_BROADCAST(peel) \ MICRO_UNROLL_WORK(func, func2, peel) \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ } @@ -2021,10 +2021,11 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs #define MICRO_ONE4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_ONE, 1) -#define MICRO_DST_PTR_ONE(iter) \ - EIGEN_IF_CONSTEXPR(unroll_factor > iter) { bsetzero(accZero##iter); } \ - else { \ - EIGEN_UNUSED_VARIABLE(accZero##iter); \ +#define MICRO_DST_PTR_ONE(iter) \ + EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ + bsetzero(accZero##iter); \ + } else { \ + EIGEN_UNUSED_VARIABLE(accZero##iter); \ } #define MICRO_DST_PTR MICRO_UNROLL(MICRO_DST_PTR_ONE) @@ -2035,13 +2036,12 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs #ifdef USE_PARTIAL_PACKETS #define MICRO_STORE_ONE(iter) \ - EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ - EIGEN_IF_CONSTEXPR(MICRO_NORMAL_PARTIAL(iter)) { \ + EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR (MICRO_NORMAL_PARTIAL(iter)) { \ bload(acc, res, row + iter * accCols, 0); \ bscale(acc, accZero##iter, pAlpha); \ bstore(acc, res, row + iter * accCols); \ - } \ - else { \ + } else { \ bload_partial(acc, res, row + iter * accCols, accCols2); \ bscale(acc, accZero##iter, pAlpha); \ bstore_partial(acc, res, row + iter * accCols, accCols2); \ @@ -2049,7 +2049,7 @@ EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs } #else #define MICRO_STORE_ONE(iter) \ - EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ bload(acc, res, row + iter * accCols, 0); \ bscale(acc, accZero##iter, pAlpha, pMask); \ bstore(acc, res, row + iter * accCols); \ @@ -2221,48 +2221,49 @@ EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const #define MICRO_COMPLEX_UNROLL(func) func(0) func(1) func(2) func(3) -#define MICRO_COMPLEX_ZERO_PEEL(peel) \ - EIGEN_IF_CONSTEXPR((PEEL_COMPLEX_ROW > peel) && (peel != 0)) { \ - bsetzero(accReal##peel); \ - bsetzero(accImag##peel); \ - } \ - else { \ - EIGEN_UNUSED_VARIABLE(accReal##peel); \ - EIGEN_UNUSED_VARIABLE(accImag##peel); \ +#define MICRO_COMPLEX_ZERO_PEEL(peel) \ + EIGEN_IF_CONSTEXPR ((PEEL_COMPLEX_ROW > peel) && (peel != 0)) { \ + bsetzero(accReal##peel); \ + bsetzero(accImag##peel); \ + } else { \ + EIGEN_UNUSED_VARIABLE(accReal##peel); \ + EIGEN_UNUSED_VARIABLE(accImag##peel); \ } -#define MICRO_COMPLEX_ADD_ROWS(N, used) \ - MICRO_ADD(ptr_real, N) \ - EIGEN_IF_CONSTEXPR(!RhsIsReal) { MICRO_ADD(ptr_imag, N) } \ - else if (used) { \ - EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \ - EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \ - EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \ +#define MICRO_COMPLEX_ADD_ROWS(N, used) \ + MICRO_ADD(ptr_real, N) \ + EIGEN_IF_CONSTEXPR (!RhsIsReal) { \ + MICRO_ADD(ptr_imag, N) \ + } else if (used) { \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \ + EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \ } -#define MICRO_COMPLEX_BROADCAST(peel) \ - MICRO_BROADCAST1(peel, ptr_real, rhsV, false) \ - EIGEN_IF_CONSTEXPR(!RhsIsReal) { MICRO_BROADCAST1(peel, ptr_imag, rhsVi, false) } \ - else { \ - EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ +#define MICRO_COMPLEX_BROADCAST(peel) \ + MICRO_BROADCAST1(peel, ptr_real, rhsV, false) \ + EIGEN_IF_CONSTEXPR (!RhsIsReal) { \ + MICRO_BROADCAST1(peel, ptr_imag, rhsVi, false) \ + } else { \ + EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } -#define MICRO_COMPLEX_BROADCAST_EXTRA \ - Packet rhsV[4], rhsVi[4]; \ - MICRO_BROADCAST_EXTRA1(ptr_real, rhsV, false) \ - EIGEN_IF_CONSTEXPR(!RhsIsReal) { MICRO_BROADCAST_EXTRA1(ptr_imag, rhsVi, false) } \ - else { \ - EIGEN_UNUSED_VARIABLE(rhsVi); \ - } \ +#define MICRO_COMPLEX_BROADCAST_EXTRA \ + Packet rhsV[4], rhsVi[4]; \ + MICRO_BROADCAST_EXTRA1(ptr_real, rhsV, false) \ + EIGEN_IF_CONSTEXPR (!RhsIsReal) { \ + MICRO_BROADCAST_EXTRA1(ptr_imag, rhsVi, false) \ + } else { \ + EIGEN_UNUSED_VARIABLE(rhsVi); \ + } \ MICRO_COMPLEX_ADD_ROWS(1, true) #define MICRO_COMPLEX_SRC2_PTR \ MICRO_SRC2(ptr_real, strideB* advanceCols, 0) \ - EIGEN_IF_CONSTEXPR(!RhsIsReal) { \ + EIGEN_IF_CONSTEXPR (!RhsIsReal) { \ MICRO_RHS(ptr_imag, 0) = rhs_base + MICRO_NEW_ROWS * strideB; \ MICRO_SRC2(ptr_imag, strideB* advanceCols, strideB) \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \ EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \ EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \ @@ -2271,22 +2272,22 @@ EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const #define MICRO_COMPLEX_ZERO_PEEL_ROW MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_ZERO_PEEL) #define MICRO_COMPLEX_WORK_PEEL(peel) \ - EIGEN_IF_CONSTEXPR(PEEL_COMPLEX_ROW > peel) { \ + EIGEN_IF_CONSTEXPR (PEEL_COMPLEX_ROW > peel) { \ MICRO_COMPLEX_BROADCAST(peel) \ pgerc( \ &accReal##peel, &accImag##peel, lhs_ptr_real + (remaining_rows * peel), \ lhs_ptr_imag + (remaining_rows * peel), rhsV##peel, rhsVi##peel); \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } -#define MICRO_COMPLEX_ADD_COLS(size) \ - lhs_ptr_real += (remaining_rows * size); \ - EIGEN_IF_CONSTEXPR(!LhsIsReal) \ - lhs_ptr_imag += (remaining_rows * size); \ - else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); +#define MICRO_COMPLEX_ADD_COLS(size) \ + lhs_ptr_real += (remaining_rows * size); \ + EIGEN_IF_CONSTEXPR (!LhsIsReal) \ + lhs_ptr_imag += (remaining_rows * size); \ + else \ + EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); #define MICRO_COMPLEX_WORK_PEEL_ROW \ Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4]; \ @@ -2296,7 +2297,7 @@ EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const MICRO_COMPLEX_ADD_ROWS(PEEL_COMPLEX_ROW, false) #define MICRO_COMPLEX_ADD_PEEL(peel, sum) \ - EIGEN_IF_CONSTEXPR(PEEL_COMPLEX_ROW > peel) { \ + EIGEN_IF_CONSTEXPR (PEEL_COMPLEX_ROW > peel) { \ for (Index i = 0; i < accRows; i++) { \ accReal##sum.packet[i] += accReal##peel.packet[i]; \ accImag##sum.packet[i] += accImag##peel.packet[i]; \ @@ -2332,9 +2333,10 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper& r const Scalar *rhs_ptr_imag0 = NULL, *rhs_ptr_imag1 = NULL, *rhs_ptr_imag2 = NULL; const Scalar* lhs_ptr_real = lhs_base + advanceRows * row * strideA + remaining_rows * offsetA; const Scalar* lhs_ptr_imag = NULL; - EIGEN_IF_CONSTEXPR(!LhsIsReal) - lhs_ptr_imag = lhs_ptr_real + remaining_rows * strideA; - else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); + EIGEN_IF_CONSTEXPR (!LhsIsReal) + lhs_ptr_imag = lhs_ptr_real + remaining_rows * strideA; + else + EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); PacketBlock accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3; PacketBlock taccReal, taccImag; PacketBlock acc0, acc1; @@ -2352,7 +2354,9 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper& r do { MICRO_COMPLEX_PREFETCHN(accRows) EIGEN_POWER_PREFETCH(lhs_ptr_real); - EIGEN_IF_CONSTEXPR(!LhsIsReal) { EIGEN_POWER_PREFETCH(lhs_ptr_imag); } + EIGEN_IF_CONSTEXPR (!LhsIsReal) { + EIGEN_POWER_PREFETCH(lhs_ptr_imag); + } MICRO_COMPLEX_WORK_PEEL_ROW } while ((k += PEEL_COMPLEX_ROW) + PEEL_COMPLEX_ROW <= remaining_depth); MICRO_COMPLEX_ADD_PEEL_ROW @@ -2369,7 +2373,9 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper& r bscalec(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); bcouple(taccReal, taccImag, tRes, acc0, acc1); bstore(acc0, res, row + 0); - EIGEN_IF_CONSTEXPR(full) { bstore(acc1, res, row + accColsC); } + EIGEN_IF_CONSTEXPR (full) { + bstore(acc1, res, row + accColsC); + } } else { bscalec(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); bcouple(taccReal, taccImag, tRes, acc0, acc1); @@ -2380,7 +2386,7 @@ EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper& r } } else { bstore(acc0, res, row + 0); - EIGEN_IF_CONSTEXPR(full) { + EIGEN_IF_CONSTEXPR (full) { for (Index j = 0; j < accRows; j++) { res(row + accColsC, j) = pfirst(acc1.packet[j]); } @@ -2408,19 +2414,18 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(const DataMapper& res, const Sca func(0, peel) func(1, peel) func(2, peel) func(3, peel) #define MICRO_COMPLEX_WORK_ONE4(iter, peel) \ - EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ pgerc_common( \ &accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \ } #define MICRO_COMPLEX_TYPE_PEEL4(func, func2, peel) \ - EIGEN_IF_CONSTEXPR(PEEL_COMPLEX > peel) { \ + EIGEN_IF_CONSTEXPR (PEEL_COMPLEX > peel) { \ Packet lhsV0, lhsV1, lhsV2, lhsV3; \ Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3; \ MICRO_COMPLEX_BROADCAST(peel) \ MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(rhsV##peel); \ EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ } @@ -2443,11 +2448,10 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(const DataMapper& res, const Sca #define MICRO_COMPLEX_ONE4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1) #define MICRO_COMPLEX_DST_PTR_ONE(iter) \ - EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ bsetzero(accReal##iter); \ bsetzero(accImag##iter); \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(accReal##iter); \ EIGEN_UNUSED_VARIABLE(accImag##iter); \ } @@ -2459,14 +2463,16 @@ EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(const DataMapper& res, const Sca #define MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_PREFETCH_ONE) #define MICRO_COMPLEX_STORE_ONE(iter) \ - EIGEN_IF_CONSTEXPR(unroll_factor > iter) { \ + EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ constexpr bool full = ((MICRO_NORMAL(iter)) || (accCols2 > accColsC)); \ bload(tRes, res, row + iter * accCols, 0); \ bscalec(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, \ taccImag, pMask); \ bcouple(taccReal, taccImag, tRes, acc0, acc1); \ bstore(acc0, res, row + iter * accCols + 0); \ - EIGEN_IF_CONSTEXPR(full) { bstore(acc1, res, row + iter * accCols + accColsC); } \ + EIGEN_IF_CONSTEXPR (full) { \ + bstore(acc1, res, row + iter * accCols + accColsC); \ + } \ } #define MICRO_COMPLEX_STORE MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_STORE_ONE) @@ -2632,8 +2638,9 @@ EIGEN_ALWAYS_INLINE Packet4f loadAndMultiplyF32(Packet4f acc, const Packet4f pAl template EIGEN_ALWAYS_INLINE void storeF32(float*& result, Packet4f result_block, Index rows, Index extra_rows) { - EIGEN_IF_CONSTEXPR(lhsExtraRows) { pstoreu_partial(result, result_block, extra_rows); } - else { + EIGEN_IF_CONSTEXPR (lhsExtraRows) { + pstoreu_partial(result, result_block, extra_rows); + } else { pstoreu(result, result_block); } result += rows; @@ -2643,13 +2650,12 @@ template EIGEN_ALWAYS_INLINE void storeResults(Packet4f (&acc)[4], Index rows, const Packet4f pAlpha, float* result, Index extra_cols, Index extra_rows) { Index x = 0; - EIGEN_IF_CONSTEXPR(rhsExtraCols) { + EIGEN_IF_CONSTEXPR (rhsExtraCols) { do { Packet4f result_block = loadAndMultiplyF32(acc[x], pAlpha, result); storeF32(result, result_block, rows, extra_rows); } while (++x < extra_cols); - } - else { + } else { Packet4f result_block[4]; float* result2 = result; do { @@ -2683,18 +2689,23 @@ EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Lo(Packet8us data) { template EIGEN_ALWAYS_INLINE void storeConvertTwoBF16(float* to, PacketBlock& block, Index extra = 0) { - EIGEN_IF_CONSTEXPR(N < 4) { pstoreu_partial(to + 0, oneConvertBF16Hi(block.packet[0].m_val), extra); } - else EIGEN_IF_CONSTEXPR(N >= (M * 8 + 4)) { + EIGEN_IF_CONSTEXPR (N < 4) { + pstoreu_partial(to + 0, oneConvertBF16Hi(block.packet[0].m_val), extra); + } else EIGEN_IF_CONSTEXPR (N >= (M * 8 + 4)) { pstoreu(to + 0, oneConvertBF16Hi(block.packet[M].m_val)); - EIGEN_IF_CONSTEXPR(N >= 8) { pstoreu(to + 4, oneConvertBF16Lo(block.packet[M].m_val)); } + EIGEN_IF_CONSTEXPR (N >= 8) { + pstoreu(to + 4, oneConvertBF16Lo(block.packet[M].m_val)); + } } } template EIGEN_ALWAYS_INLINE void storeConvertBlockBF16(float* to, PacketBlock& block, Index extra) { storeConvertTwoBF16(to + 0, block, extra); - EIGEN_IF_CONSTEXPR(N >= 16) { storeConvertTwoBF16(to + 8, block); } - EIGEN_IF_CONSTEXPR(N >= 32) { + EIGEN_IF_CONSTEXPR (N >= 16) { + storeConvertTwoBF16(to + 8, block); + } + EIGEN_IF_CONSTEXPR (N >= 32) { storeConvertTwoBF16(to + 16, block); storeConvertTwoBF16(to + 24, block); } @@ -2702,8 +2713,9 @@ EIGEN_ALWAYS_INLINE void storeConvertBlockBF16(float* to, PacketBlock EIGEN_ALWAYS_INLINE Packet8bf loadBF16fromResult(bfloat16* src, Index resInc) { - EIGEN_IF_CONSTEXPR(non_unit_stride) { return pgather(src + delta * resInc, resInc); } - else { + EIGEN_IF_CONSTEXPR (non_unit_stride) { + return pgather(src + delta * resInc, resInc); + } else { return ploadu(src + delta); } } @@ -2745,7 +2757,7 @@ EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32DupOne(float* result, Index } for (Index j = 0; j < 4 * size; j += 4) { - EIGEN_IF_CONSTEXPR(lhsExtraRows) { + EIGEN_IF_CONSTEXPR (lhsExtraRows) { Packet4f z = pset1(float(0)); Index i = 0; do { @@ -2754,8 +2766,7 @@ EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32DupOne(float* result, Index do { pstoreu(result + (j + i) * 4, z); } while (++i < 4); - } - else { + } else { for (Index i = 0; i < 4; i++) { pstoreu(result + (j + i) * 4, dup[j + i]); } @@ -2785,15 +2796,17 @@ EIGEN_ALWAYS_INLINE void convertPointerBF16toF32(Index& i, float* result, Index while (i + size <= rows) { PacketBlock r32; r32.packet[0] = loadBF16fromResult(src, resInc); - EIGEN_IF_CONSTEXPR(size >= 16) { r32.packet[1] = loadBF16fromResult(src, resInc); } - EIGEN_IF_CONSTEXPR(size >= 32) { + EIGEN_IF_CONSTEXPR (size >= 16) { + r32.packet[1] = loadBF16fromResult(src, resInc); + } + EIGEN_IF_CONSTEXPR (size >= 32) { r32.packet[2] = loadBF16fromResult(src, resInc); r32.packet[3] = loadBF16fromResult(src, resInc); } storeConvertBlockBF16(result + i, r32, rows & 3); i += extra; src += extra * resInc; - EIGEN_IF_CONSTEXPR(size != 32) break; + EIGEN_IF_CONSTEXPR (size != 32) break; } } @@ -2859,7 +2872,7 @@ EIGEN_ALWAYS_INLINE void outputResultsVSX(Packet4f (&acc)[num_acc][4], Index row for (Index i = 0; i < real_rhs; i++, result += 4 * rows, k++) { storeResults(acc[k], rows, pAlpha, result, extra_cols, extra_rows); } - EIGEN_IF_CONSTEXPR(rhsExtraCols) { + EIGEN_IF_CONSTEXPR (rhsExtraCols) { storeResults(acc[k], rows, pAlpha, result, extra_cols, extra_rows); } } @@ -2867,12 +2880,11 @@ EIGEN_ALWAYS_INLINE void outputResultsVSX(Packet4f (&acc)[num_acc][4], Index row template EIGEN_ALWAYS_INLINE void loadTwoRhsFloat32(const float* block, Index strideB, Index i, Packet4f& dhs0, Packet4f& dhs1) { dhs0 = ploadu(block + strideB * i + 0); - EIGEN_IF_CONSTEXPR(zero) { + EIGEN_IF_CONSTEXPR (zero) { Packet4f dhs2 = pset1(float(0)); dhs1 = vec_mergel(dhs0, dhs2); dhs0 = vec_mergeh(dhs0, dhs2); - } - else { + } else { dhs1 = ploadu(block + strideB * i + 4); } } @@ -2887,7 +2899,7 @@ EIGEN_ALWAYS_INLINE void KLoop(const float* indexA, const float* indexB, Packet4 for (Index i = 0; i < real_rhs; i += 2) { loadTwoRhsFloat32(indexB + k * 4, strideB, i, rhs[i + 0], rhs[i + 1]); } - EIGEN_IF_CONSTEXPR(rhsExtraCols) { + EIGEN_IF_CONSTEXPR (rhsExtraCols) { loadTwoRhsFloat32(indexB + k * extra_cols - offsetB, strideB, real_rhs, rhs[real_rhs + 0], rhs[real_rhs + 1]); } @@ -2948,7 +2960,7 @@ template EIGEN_ALWAYS_INLINE void colVSXLoopBodyExtraN(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const float* indexA, const float* blockB, Index strideB, Index offsetB, float* result) { - EIGEN_IF_CONSTEXPR(MAX_BFLOAT16_ACC_VSX > num_acc) { + EIGEN_IF_CONSTEXPR (MAX_BFLOAT16_ACC_VSX > num_acc) { colVSXLoopBody(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); } @@ -2971,7 +2983,7 @@ void colVSXLoopBodyExtra(Index col, Index depth, Index cols, Index rows, const P offsetB, result); break; default: - EIGEN_IF_CONSTEXPR(rhsExtraCols) { + EIGEN_IF_CONSTEXPR (rhsExtraCols) { colVSXLoopBody<1, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); } break; @@ -3025,14 +3037,16 @@ EIGEN_ALWAYS_INLINE void convertBF16toF32(Index& i, float* result, Index rows, c while (i + size <= rows) { PacketBlock r32; r32.packet[0] = src.template loadPacket(i + 0); - EIGEN_IF_CONSTEXPR(size >= 16) { r32.packet[1] = src.template loadPacket(i + 8); } - EIGEN_IF_CONSTEXPR(size >= 32) { + EIGEN_IF_CONSTEXPR (size >= 16) { + r32.packet[1] = src.template loadPacket(i + 8); + } + EIGEN_IF_CONSTEXPR (size >= 32) { r32.packet[2] = src.template loadPacket(i + 16); r32.packet[3] = src.template loadPacket(i + 24); } storeConvertBlockBF16(result + i, r32, rows & 3); i += extra; - EIGEN_IF_CONSTEXPR(size != 32) break; + EIGEN_IF_CONSTEXPR (size != 32) break; } } diff --git a/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc b/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc index b3a90c640..418c00f9e 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc +++ b/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc @@ -91,18 +91,24 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal func(0, N, which) func(1, N, which) func(2, N, which) func(3, N, which) func(4, N, which) func(5, N, which) \ func(6, N, which) func(7, N, which) -#define GEMV_UNUSED_VAR(iter, N, which) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) <= iter) { EIGEN_UNUSED_VARIABLE(which##iter); } +#define GEMV_UNUSED_VAR(iter, N, which) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) <= iter) { \ + EIGEN_UNUSED_VARIABLE(which##iter); \ + } #define GEMV_UNUSED_EXTRA_VAR(iter, N, which) \ - EIGEN_IF_CONSTEXPR(N <= iter) { EIGEN_UNUSED_VARIABLE(which##iter); } + EIGEN_IF_CONSTEXPR (N <= iter) { \ + EIGEN_UNUSED_VARIABLE(which##iter); \ + } #define GEMV_UNUSED_EXTRA(N, which) GEMV_UNROLL3(GEMV_UNUSED_EXTRA_VAR, N, which) #define GEMV_UNUSED(N, which) GEMV_UNROLL3(GEMV_UNUSED_VAR, N, which) -#define GEMV_INIT_MMA(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { __builtin_mma_xxsetaccz(&e##iter); } +#define GEMV_INIT_MMA(iter, N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \ + __builtin_mma_xxsetaccz(&e##iter); \ + } #if EIGEN_COMP_LLVM #define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \ @@ -114,41 +120,38 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal #endif #define GEMV_LOAD1A_COL_MMA(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ g##iter = GEMV_LOADPACKET_COL(iter); \ EIGEN_UNUSED_VARIABLE(b##iter); \ - } \ - else { \ + } else { \ GEMV_LOADPAIR_COL_MMA(iter, iter << 1) \ EIGEN_UNUSED_VARIABLE(g##iter); \ } \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(b##iter); \ EIGEN_UNUSED_VARIABLE(g##iter); \ } -#define GEMV_WORK1A_COL_MMA(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { pger_vecMMA_acc(&e##iter, a0, g##iter); } \ - else { \ - pger_vecMMA_acc(&e##iter, b##iter, a0); \ - } \ +#define GEMV_WORK1A_COL_MMA(iter, N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ + pger_vecMMA_acc(&e##iter, a0, g##iter); \ + } else { \ + pger_vecMMA_acc(&e##iter, b##iter, a0); \ + } \ } #define GEMV_LOAD1B_COL_MMA(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ GEMV_LOADPAIR_COL_MMA(iter2, iter2) \ EIGEN_UNUSED_VARIABLE(b##iter3); \ - } \ - else { \ + } else { \ GEMV_LOADPAIR_COL_MMA(iter2, iter2 << 1) \ GEMV_LOADPAIR_COL_MMA(iter3, iter3 << 1) \ } \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(b##iter2); \ EIGEN_UNUSED_VARIABLE(b##iter3); \ } \ @@ -156,30 +159,31 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal EIGEN_UNUSED_VARIABLE(g##iter3); #define GEMV_WORK1B_COL_MMA(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ LhsPacket h[2]; \ __builtin_vsx_disassemble_pair(reinterpret_cast(h), &b##iter2); \ pger_vecMMA_acc(&e##iter2, a0, h[0]); \ pger_vecMMA_acc(&e##iter3, a0, h[1]); \ - } \ - else { \ + } else { \ pger_vecMMA_acc(&e##iter2, b##iter2, a0); \ pger_vecMMA_acc(&e##iter3, b##iter3, a0); \ } \ } #if EIGEN_COMP_LLVM -#define GEMV_LOAD_COL_MMA(N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > 1) { GEMV_UNROLL_HALF(GEMV_LOAD1B_COL_MMA, (N >> 1)) } \ - else { \ - GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) \ +#define GEMV_LOAD_COL_MMA(N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_LOAD1B_COL_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) \ } -#define GEMV_WORK_COL_MMA(N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > 1) { GEMV_UNROLL_HALF(GEMV_WORK1B_COL_MMA, (N >> 1)) } \ - else { \ - GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) \ +#define GEMV_WORK_COL_MMA(N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_WORK1B_COL_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) \ } #else #define GEMV_LOAD_COL_MMA(N) GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) @@ -188,9 +192,9 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal #endif #define GEMV_DISASSEMBLE_MMA(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \ __builtin_mma_disassemble_acc(&result##iter.packet, &e##iter); \ - EIGEN_IF_CONSTEXPR(!GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR (!GEMV_IS_FLOAT) { \ result##iter.packet[0][1] = result##iter.packet[1][0]; \ result##iter.packet[2][1] = result##iter.packet[3][0]; \ } \ @@ -200,17 +204,15 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal b##iter1 = *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize)); #define GEMV_LOAD2_COL_MMA(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ GEMV_LOADPAIR2_COL_MMA(iter2, iter2); \ EIGEN_UNUSED_VARIABLE(b##iter3); \ - } \ - else { \ + } else { \ GEMV_LOADPAIR2_COL_MMA(iter2, iter2 << 1); \ GEMV_LOADPAIR2_COL_MMA(iter3, iter3 << 1); \ } \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(b##iter2); \ EIGEN_UNUSED_VARIABLE(b##iter3); \ } @@ -224,88 +226,95 @@ EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScal GEMV_BUILDPAIR_MMA(b##iter2, f##iter2[0], f##iter2[1]); #else #define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ __asm__("xvmaddasp %0,%x1,%x3\n\txvmaddasp %L0,%x2,%x3" \ : "+&d"(b##iter2) \ : "wa"(result##iter3.packet[0]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \ - } \ - else { \ + } else { \ __asm__("xvmaddadp %0,%x1,%x3\n\txvmaddadp %L0,%x2,%x3" \ : "+&d"(b##iter2) \ : "wa"(result##iter2.packet[2]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \ } #endif -#define GEMV_WORK2_COL_MMA(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter2); } \ - else { \ - GEMV_WORKPAIR2_COL_MMA(iter2, iter2, iter2 << 1); \ - GEMV_WORKPAIR2_COL_MMA(iter3, iter3, iter3 << 1); \ - } \ +#define GEMV_WORK2_COL_MMA(iter1, iter2, iter3, N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ + GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter2); \ + } else { \ + GEMV_WORKPAIR2_COL_MMA(iter2, iter2, iter2 << 1); \ + GEMV_WORKPAIR2_COL_MMA(iter3, iter3, iter3 << 1); \ + } \ } #define GEMV_STOREPAIR2_COL_MMA(iter1, iter2) \ *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize)) = b##iter1; #define GEMV_STORE_COL_MMA(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ storeMaddData(res + i + (iter * ResPacketSize), palpha, result##iter.packet[0]); \ - } \ - else { \ + } else { \ GEMV_LOADPAIR2_COL_MMA(iter, iter << 1) \ GEMV_WORKPAIR2_COL_MMA(iter, iter, iter << 1) \ GEMV_STOREPAIR2_COL_MMA(iter, iter << 1) \ } \ } -#define GEMV_STORE2_COL_MMA(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter1) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { GEMV_STOREPAIR2_COL_MMA(iter2, iter2); } \ - else { \ - GEMV_STOREPAIR2_COL_MMA(iter2, iter2 << 1) \ - GEMV_STOREPAIR2_COL_MMA(iter3, iter3 << 1) \ - } \ +#define GEMV_STORE2_COL_MMA(iter1, iter2, iter3, N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ + GEMV_STOREPAIR2_COL_MMA(iter2, iter2); \ + } else { \ + GEMV_STOREPAIR2_COL_MMA(iter2, iter2 << 1) \ + GEMV_STOREPAIR2_COL_MMA(iter3, iter3 << 1) \ + } \ } -#define GEMV_PROCESS_COL_ONE_MMA(N) \ - GEMV_UNROLL(GEMV_INIT_MMA, N) \ - Index j = j2; \ - __vector_pair b0, b1, b2, b3, b4, b5, b6, b7; \ - do { \ - LhsPacket g0, g1, g2, g3, g4, g5, g6, g7; \ - RhsPacket a0 = pset1(rhs2(j, 0)); \ - GEMV_UNROLL(GEMV_PREFETCH, N) \ - GEMV_LOAD_COL_MMA(N) \ - GEMV_WORK_COL_MMA(N) \ - } while (++j < jend); \ - GEMV_UNROLL(GEMV_DISASSEMBLE_MMA, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) <= 1) { GEMV_UNROLL(GEMV_STORE_COL_MMA, N) } \ - else {GEMV_UNROLL_HALF(GEMV_LOAD2_COL_MMA, (N >> 1)) GEMV_UNROLL_HALF(GEMV_WORK2_COL_MMA, (N >> 1)) \ - GEMV_UNROLL_HALF(GEMV_STORE2_COL_MMA, (N >> 1))} i += (ResPacketSize * N); +#define GEMV_PROCESS_COL_ONE_MMA(N) \ + GEMV_UNROLL(GEMV_INIT_MMA, N) \ + Index j = j2; \ + __vector_pair b0, b1, b2, b3, b4, b5, b6, b7; \ + do { \ + LhsPacket g0, g1, g2, g3, g4, g5, g6, g7; \ + RhsPacket a0 = pset1(rhs2(j, 0)); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_LOAD_COL_MMA(N) \ + GEMV_WORK_COL_MMA(N) \ + } while (++j < jend); \ + GEMV_UNROLL(GEMV_DISASSEMBLE_MMA, N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) <= 1) { \ + GEMV_UNROLL(GEMV_STORE_COL_MMA, N) \ + } else { \ + GEMV_UNROLL_HALF(GEMV_LOAD2_COL_MMA, (N >> 1)) \ + GEMV_UNROLL_HALF(GEMV_WORK2_COL_MMA, (N >> 1)) GEMV_UNROLL_HALF(GEMV_STORE2_COL_MMA, (N >> 1)) \ + } \ + i += (ResPacketSize * N); #endif -#define GEMV_INIT(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { c##iter = pset1(ResScalar(0)); } \ - else { \ - EIGEN_UNUSED_VARIABLE(c##iter); \ +#define GEMV_INIT(iter, N) \ + EIGEN_IF_CONSTEXPR (N > iter) { \ + c##iter = pset1(ResScalar(0)); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c##iter); \ } #ifdef EIGEN_POWER_USE_GEMV_PREFETCH -#define GEMV_PREFETCH(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > ((iter >> 1) + ((N >> 1) * (iter & 1)))) { \ - lhs.prefetch(i + (iter * LhsPacketSize) + prefetch_dist, j); \ +#define GEMV_PREFETCH(iter, N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > ((iter >> 1) + ((N >> 1) * (iter & 1)))) { \ + lhs.prefetch(i + (iter * LhsPacketSize) + prefetch_dist, j); \ } #else #define GEMV_PREFETCH(iter, N) #endif -#define GEMV_WORK_COL(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { c##iter = pcj.pmadd(GEMV_LOADPACKET_COL(iter), a0, c##iter); } +#define GEMV_WORK_COL(iter, N) \ + EIGEN_IF_CONSTEXPR (N > iter) { \ + c##iter = pcj.pmadd(GEMV_LOADPACKET_COL(iter), a0, c##iter); \ + } #define GEMV_STORE_COL(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { \ + EIGEN_IF_CONSTEXPR (N > iter) { \ pstoreu(res + i + (iter * ResPacketSize), \ pmadd(c##iter, palpha, ploadu(res + i + (iter * ResPacketSize)))); \ } @@ -437,8 +446,9 @@ template EIGEN_ALWAYS_INLINE void outputVecCol(Packet4f acc, float* result, Packet4f pAlpha, Index extra_rows) { Packet4f d0 = ploadu(result); d0 = pmadd(acc, pAlpha, d0); - EIGEN_IF_CONSTEXPR(extraRows) { pstoreu_partial(result, d0, extra_rows); } - else { + EIGEN_IF_CONSTEXPR (extraRows) { + pstoreu_partial(result, d0, extra_rows); + } else { pstoreu(result, d0); } } @@ -450,7 +460,9 @@ EIGEN_ALWAYS_INLINE void outputVecColResults(Packet4f (&acc)[num_acc][size], flo for (Index k = 0; k < real_acc; k++) { outputVecCol(acc[k][0], result + k * 4, pAlpha, extra_rows); } - EIGEN_IF_CONSTEXPR(extraRows) { outputVecCol(acc[real_acc][0], result + real_acc * 4, pAlpha, extra_rows); } + EIGEN_IF_CONSTEXPR (extraRows) { + outputVecCol(acc[real_acc][0], result + real_acc * 4, pAlpha, extra_rows); + } } static Packet16uc p16uc_MERGE16_32_V1 = {0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17}; @@ -460,16 +472,18 @@ template EIGEN_ALWAYS_INLINE void loadVecLoopVSX(Index k, LhsMapper& lhs, Packet4f (&a0)[num_acc][2]) { Packet8bf c0 = lhs.template loadPacket(k * 4, 0); Packet8bf b1; - EIGEN_IF_CONSTEXPR(!zero) { + EIGEN_IF_CONSTEXPR (!zero) { b1 = lhs.template loadPacket(k * 4, 1); a0[k + 0][1] = oneConvertBF16Hi(b1.m_val); } a0[k + 0][0] = oneConvertBF16Hi(c0.m_val); - EIGEN_IF_CONSTEXPR(num_acc > (k + 1)) { + EIGEN_IF_CONSTEXPR (num_acc > (k + 1)) { a0[k + 1][0] = oneConvertBF16Lo(c0.m_val); - EIGEN_IF_CONSTEXPR(!zero) { a0[k + 1][1] = oneConvertBF16Lo(b1.m_val); } + EIGEN_IF_CONSTEXPR (!zero) { + a0[k + 1][1] = oneConvertBF16Lo(b1.m_val); + } } } @@ -515,7 +529,9 @@ EIGEN_ALWAYS_INLINE void vecColLoopVSX(Index j, LhsMapper& lhs, RhsMapper& rhs, Packet8bf b2 = loadColData(rhs, j); b0[0] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V1); - EIGEN_IF_CONSTEXPR(!zero) { b0[1] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V2); } + EIGEN_IF_CONSTEXPR (!zero) { + b0[1] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V2); + } using LhsSubMapper = typename LhsMapper::SubMapper; @@ -570,7 +586,7 @@ void colVSXVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, Rh template EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float* result) { - EIGEN_IF_CONSTEXPR(MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { + EIGEN_IF_CONSTEXPR (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { colVSXVecColLoopBody(row, cend, rows, lhs, rhs, pAlpha, result); } @@ -602,7 +618,7 @@ EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtra(Index& row, Index cend, Index colVSXVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); break; default: - EIGEN_IF_CONSTEXPR(extraRows) { + EIGEN_IF_CONSTEXPR (extraRows) { colVSXVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result); } break; @@ -628,13 +644,15 @@ EIGEN_ALWAYS_INLINE void calcVSXVecColLoops(Index cend, Index rows, LhsMapper& l template EIGEN_ALWAYS_INLINE void storeBF16fromResult(bfloat16* dst, Packet8bf data, Index resInc, Index extra) { if (inc) { - EIGEN_IF_CONSTEXPR(size < 8) { pscatter_partial(dst + delta * resInc, data, resInc, extra); } - else { + EIGEN_IF_CONSTEXPR (size < 8) { + pscatter_partial(dst + delta * resInc, data, resInc, extra); + } else { pscatter(dst + delta * resInc, data, resInc); } } else { - EIGEN_IF_CONSTEXPR(size < 8) { pstoreu_partial(dst + delta, data, extra); } - else { + EIGEN_IF_CONSTEXPR (size < 8) { + pstoreu_partial(dst + delta, data, extra); + } else { pstoreu(dst + delta, data); } } @@ -647,20 +665,24 @@ EIGEN_ALWAYS_INLINE void convertPointerF32toBF16VSX(Index& i, float* result, Ind while (i + size <= rows) { PacketBlock r32; r32.packet[0] = convertF32toBF16VSX(result + i + 0); - EIGEN_IF_CONSTEXPR(size >= 16) { r32.packet[1] = convertF32toBF16VSX(result + i + 8); } - EIGEN_IF_CONSTEXPR(size >= 32) { + EIGEN_IF_CONSTEXPR (size >= 16) { + r32.packet[1] = convertF32toBF16VSX(result + i + 8); + } + EIGEN_IF_CONSTEXPR (size >= 32) { r32.packet[2] = convertF32toBF16VSX(result + i + 16); r32.packet[3] = convertF32toBF16VSX(result + i + 24); } storeBF16fromResult(dst, r32.packet[0], resInc, rows & 7); - EIGEN_IF_CONSTEXPR(size >= 16) { storeBF16fromResult(dst, r32.packet[1], resInc); } - EIGEN_IF_CONSTEXPR(size >= 32) { + EIGEN_IF_CONSTEXPR (size >= 16) { + storeBF16fromResult(dst, r32.packet[1], resInc); + } + EIGEN_IF_CONSTEXPR (size >= 32) { storeBF16fromResult(dst, r32.packet[2], resInc); storeBF16fromResult(dst, r32.packet[3], resInc); } i += extra; dst += extra * resInc; - EIGEN_IF_CONSTEXPR(size != 32) break; + EIGEN_IF_CONSTEXPR (size != 32) break; } } @@ -743,8 +765,9 @@ EIGEN_ALWAYS_INLINE void outputVecResults(Packet4f (&acc)[num_acc][size], float* Packet4f d0 = ploadu(result + k); d0 = pmadd(acc[k + 0][0], pAlpha, d0); - EIGEN_IF_CONSTEXPR(num_acc > (k + 3)) { pstoreu(result + k, d0); } - else { + EIGEN_IF_CONSTEXPR (num_acc > (k + 3)) { + pstoreu(result + k, d0); + } else { if (extra == 3) { pstoreu_partial(result + k, d0, extra); } else { @@ -756,13 +779,12 @@ EIGEN_ALWAYS_INLINE void outputVecResults(Packet4f (&acc)[num_acc][size], float* template EIGEN_ALWAYS_INLINE void preduxVecResults2VSX(Packet4f (&acc)[num_acc][2], Index k) { - EIGEN_IF_CONSTEXPR(num_acc > (k + 1)) { + EIGEN_IF_CONSTEXPR (num_acc > (k + 1)) { acc[k][1] = vec_mergel(acc[k + 0][0], acc[k + 1][0]); acc[k][0] = vec_mergeh(acc[k + 0][0], acc[k + 1][0]); acc[k][0] = acc[k][0] + acc[k][1]; acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8); - } - else { + } else { acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8); #ifdef _BIG_ENDIAN acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12); @@ -776,7 +798,7 @@ template EIGEN_ALWAYS_INLINE void preduxVecResultsVSX(Packet4f (&acc)[num_acc][2]) { for (Index k = 0; k < num_acc; k += 4) { preduxVecResults2VSX(acc, k + 0); - EIGEN_IF_CONSTEXPR(num_acc > (k + 2)) { + EIGEN_IF_CONSTEXPR (num_acc > (k + 2)) { preduxVecResults2VSX(acc, k + 2); #ifdef EIGEN_VECTORIZE_VSX acc[k + 0][0] = reinterpret_cast( @@ -873,7 +895,7 @@ void colVSXVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMa template EIGEN_ALWAYS_INLINE void colVSXVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float* result) { - EIGEN_IF_CONSTEXPR(MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { + EIGEN_IF_CONSTEXPR (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { colVSXVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); } } @@ -1178,8 +1200,9 @@ EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine_row(std::complex* sr /** \internal load a scalar or a vector from complex location */ template EIGEN_ALWAYS_INLINE Packet4f pload_complex(std::complex* src) { - EIGEN_IF_CONSTEXPR(GEMV_IS_SCALAR) { return pload_complex_half(src); } - else { + EIGEN_IF_CONSTEXPR (GEMV_IS_SCALAR) { + return pload_complex_half(src); + } else { return ploadu(reinterpret_cast(src)); } } @@ -1240,8 +1263,9 @@ EIGEN_ALWAYS_INLINE Packet2d pload_real_full(std::complex* src) { /** \internal load a vector from a real-only scalar location - row-wise */ template EIGEN_ALWAYS_INLINE Packet4f pload_real_row(float* src) { - EIGEN_IF_CONSTEXPR(GEMV_IS_SCALAR) { return pload_real_full(src); } - else { + EIGEN_IF_CONSTEXPR (GEMV_IS_SCALAR) { + return pload_real_full(src); + } else { return ploadu(src); } } @@ -1305,11 +1329,10 @@ EIGEN_ALWAYS_INLINE Packet1cd pset_zero() { /** \internal initialize a vector from another vector */ template EIGEN_ALWAYS_INLINE Packet pset_init(Packet& c1) { - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { EIGEN_UNUSED_VARIABLE(c1); return pset_zero(); - } - else { + } else { return c1; // Intentionally left uninitialized } } @@ -1337,12 +1360,11 @@ template EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, AlphaData& b0, ResScalar* res) { PResPacket c2 = pcplxflipconj(c0); - EIGEN_IF_CONSTEXPR(GEMV_IS_SCALAR) { + EIGEN_IF_CONSTEXPR (GEMV_IS_SCALAR) { ScalarPacket c4 = ploadu(reinterpret_cast(res)); ScalarPacket c3 = pmadd_complex(c0.v, c2.v, c4, b0); pstoreu(reinterpret_cast(res), c3); - } - else { + } else { ScalarPacket c4 = pload_complex(res); PResPacket c3 = PResPacket(pmadd_complex(c0.v, c2.v, c4, b0)); pstoreu(res, c3); @@ -1370,11 +1392,10 @@ EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, PResPacket& c1, A c6[1] = PResPacket(pmadd_complex(c1.v, c3.v, c6[1].v, b0)); GEMV_BUILDPAIR_MMA(a, c6[0].v, c6[1].v); #else - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { __asm__("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.r.v), "wa"(c0.v), "wa"(c1.v)); __asm__("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.i.v), "wa"(c2.v), "wa"(c3.v)); - } - else { + } else { __asm__("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.r.v), "wa"(c0.v), "wa"(c1.v)); __asm__("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.i.v), "wa"(c2.v), "wa"(c3.v)); } @@ -1386,7 +1407,7 @@ EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, PResPacket& c1, A /** \internal load lhs packet */ template EIGEN_ALWAYS_INLINE LhsPacket loadLhsPacket(LhsMapper& lhs, Index i, Index j) { - EIGEN_IF_CONSTEXPR(sizeof(Scalar) == sizeof(LhsScalar)) { + EIGEN_IF_CONSTEXPR (sizeof(Scalar) == sizeof(LhsScalar)) { const LhsScalar& src = lhs(i + 0, j); return LhsPacket(pload_real_full(const_cast(&src))); } @@ -1396,11 +1417,11 @@ EIGEN_ALWAYS_INLINE LhsPacket loadLhsPacket(LhsMapper& lhs, Index i, Index j) { /** \internal madd for complex times complex */ template EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_complex(RealPacket& a, RealPacket& b, RealPacket& c) { - EIGEN_IF_CONSTEXPR(ConjugateLhs && ConjugateRhs) { return vec_madd(a, pconj2(ComplexPacket(b)).v, c); } - else EIGEN_IF_CONSTEXPR(Negate && !ConjugateLhs && ConjugateRhs) { + EIGEN_IF_CONSTEXPR (ConjugateLhs && ConjugateRhs) { + return vec_madd(a, pconj2(ComplexPacket(b)).v, c); + } else EIGEN_IF_CONSTEXPR (Negate && !ConjugateLhs && ConjugateRhs) { return vec_nmsub(a, b, c); - } - else { + } else { return vec_madd(a, b, c); } } @@ -1408,8 +1429,9 @@ EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_complex(RealPacket& a, RealPacket& /** \internal madd for complex times real */ template EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_real(RealPacket& a, RealPacket& b, RealPacket& c) { - EIGEN_IF_CONSTEXPR(Conjugate) { return vec_madd(a, pconj2(ComplexPacket(b)).v, c); } - else { + EIGEN_IF_CONSTEXPR (Conjugate) { + return vec_madd(a, pconj2(ComplexPacket(b)).v, c); + } else { return vec_madd(a, b, c); } } @@ -1419,8 +1441,9 @@ template pcj; RhsPacket b0; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pset1(*b); } - else { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { + b0 = pset1(*b); + } else { b0 = ploadu(b); } c0 = pcj.pmadd(a0, b0, c0); @@ -1431,11 +1454,12 @@ template EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0, ResPacket& c1) { ScalarPacket br, bi; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { pload_realimag(b, br, bi); } - else { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { + pload_realimag(b, br, bi); + } else { pload_realimag_row(b, br, bi); } - EIGEN_IF_CONSTEXPR(ConjugateLhs && !ConjugateRhs) a0 = pconj2(a0); + EIGEN_IF_CONSTEXPR (ConjugateLhs && !ConjugateRhs) a0 = pconj2(a0); LhsPacket a1 = pcplxflipconj(a0); ScalarPacket cr = pmadd_complex_complex(a0.v, br, c0.v); ScalarPacket ci = pmadd_complex_complex(a1.v, bi, c1.v); @@ -1448,8 +1472,9 @@ template EIGEN_ALWAYS_INLINE void gemv_mult_real_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0) { ScalarPacket b0; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_complex_full(b); } - else { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { + b0 = pload_complex_full(b); + } else { b0 = pload_complex_full_row(b); } ScalarPacket cri = pmadd_complex_real(a0, b0, c0.v); @@ -1462,8 +1487,9 @@ template (&a0); ScalarPacket b0; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_real(b); } - else { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { + b0 = pload_real(b); + } else { b0 = pload_real_row(b); } ScalarPacket cri = pmadd_complex_real(a1, b0, c0.v); @@ -1542,10 +1568,9 @@ EIGEN_ALWAYS_INLINE void pload_complex_MMA(__vector_pair&) { /** \internal perform a matrix multiply and accumulate (positive and negative) of packet a and packet b */ template EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, RhsPacket& a, LhsPacket& b) { - EIGEN_IF_CONSTEXPR(NegativeAccumulate) { + EIGEN_IF_CONSTEXPR (NegativeAccumulate) { __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b); - } - else { + } else { __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b); } } @@ -1553,8 +1578,9 @@ EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, RhsPacket& a, LhsPacket /** \internal perform a matrix multiply and accumulate (positive and negative) of vector_pair a and packet b */ template EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, __vector_pair& a, Packet2d& b) { - EIGEN_IF_CONSTEXPR(NegativeAccumulate) { __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b); } - else { + EIGEN_IF_CONSTEXPR (NegativeAccumulate) { + __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b); + } else { __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b); } } @@ -1567,28 +1593,24 @@ EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad*, __vector_pair&, Packet4f&) /** \internal madd for complex times complex (MMA version) */ template EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(LhsPacket& a, RealPacket& b, __vector_quad* c) { - EIGEN_IF_CONSTEXPR(ConjugateLhs && ConjugateRhs) { + EIGEN_IF_CONSTEXPR (ConjugateLhs && ConjugateRhs) { RealPacket b2 = pconj2(convertComplex(b)).v; return pger_vecMMA(c, b2, a.v); - } - else EIGEN_IF_CONSTEXPR(Negate && !ConjugateLhs && ConjugateRhs) { + } else EIGEN_IF_CONSTEXPR (Negate && !ConjugateLhs && ConjugateRhs) { return pger_vecMMA(c, b, a.v); - } - else { + } else { return pger_vecMMA(c, b, a.v); } } template EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(__vector_pair& a, RealPacket& b, __vector_quad* c) { - EIGEN_IF_CONSTEXPR(ConjugateLhs && ConjugateRhs) { + EIGEN_IF_CONSTEXPR (ConjugateLhs && ConjugateRhs) { RealPacket b2 = pconj2(convertComplex(b)).v; return pger_vecMMA(c, a, b2); - } - else EIGEN_IF_CONSTEXPR(Negate && !ConjugateLhs && ConjugateRhs) { + } else EIGEN_IF_CONSTEXPR (Negate && !ConjugateLhs && ConjugateRhs) { return pger_vecMMA(c, a, b); - } - else { + } else { return pger_vecMMA(c, a, b); } } @@ -1597,16 +1619,17 @@ EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(__vector_pair& a, RealPacket& template EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(LhsPacket& a, RealPacket& b, __vector_quad* c) { RealPacket a2 = convertReal(a); - EIGEN_IF_CONSTEXPR(Conjugate) { + EIGEN_IF_CONSTEXPR (Conjugate) { RealPacket b2 = pconj2(convertComplex(b)).v; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { return pger_vecMMA(c, b2, a2); } - else { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { + return pger_vecMMA(c, b2, a2); + } else { return pger_vecMMA(c, a2, b2); } - } - else { - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { return pger_vecMMA(c, b, a2); } - else { + } else { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { + return pger_vecMMA(c, b, a2); + } else { return pger_vecMMA(c, a2, b); } } @@ -1615,11 +1638,10 @@ EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(LhsPacket& a, RealPacket& b, __v /** \internal madd for real times complex (MMA version) */ template EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(__vector_pair& a, RealPacket& b, __vector_quad* c) { - EIGEN_IF_CONSTEXPR(Conjugate) { + EIGEN_IF_CONSTEXPR (Conjugate) { RealPacket b2 = pconj2(convertComplex(b)).v; return pger_vecMMA(c, a, b2); - } - else { + } else { return pger_vecMMA(c, a, b); } } @@ -1629,8 +1651,9 @@ template EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) { ScalarPacket b0; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_realimag_combine(b); } - else { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { + b0 = pload_realimag_combine(b); + } else { b0 = pload_realimag_combine_row(b); } pmadd_complex_complex_MMA(a0, b0, c0); @@ -1642,8 +1665,9 @@ template (a0); ScalarPacket b0; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_real(b); } - else { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { + b0 = pload_real(b); + } else { b0 = pload_real_row(b); } pmadd_complex_real_MMA(a0, b0, c0); @@ -1654,8 +1678,9 @@ template EIGEN_ALWAYS_INLINE void gemv_mult_real_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) { ScalarPacket b0; - EIGEN_IF_CONSTEXPR(StorageOrder == ColMajor) { b0 = pload_complex_full(b); } - else { + EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) { + b0 = pload_complex_full(b); + } else { b0 = pload_complex_full_row(b); } pmadd_complex_real_MMA) template EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(__vector_pair& a0, std::complex* b, __vector_quad* c0) { - EIGEN_IF_CONSTEXPR(sizeof(LhsScalar) == 16) { + EIGEN_IF_CONSTEXPR (sizeof(LhsScalar) == 16) { gemv_mult_complex_complex_MMA(a0, b, c0); - } - else { + } else { gemv_mult_real_complex_MMA(a0, b, c0); } @@ -1717,8 +1741,8 @@ template EIGEN_ALWAYS_INLINE void disassembleResults2(__vector_quad* c0, PacketBlock& result0) { __builtin_mma_disassemble_acc(&result0.packet, c0); - EIGEN_IF_CONSTEXPR(sizeof(LhsPacket) == 16) { - EIGEN_IF_CONSTEXPR(sizeof(RhsPacket) == 16) { + EIGEN_IF_CONSTEXPR (sizeof(LhsPacket) == 16) { + EIGEN_IF_CONSTEXPR (sizeof(RhsPacket) == 16) { ScalarPacket tmp0, tmp2; tmp2 = vec_mergeh(result0.packet[2], result0.packet[3]); tmp0 = vec_mergeh(result0.packet[0], result0.packet[1]); @@ -1727,22 +1751,19 @@ EIGEN_ALWAYS_INLINE void disassembleResults2(__vector_quad* c0, PacketBlock EIGEN_ALWAYS_INLINE void disassembleResults4(__vector_quad* c0, PacketBlock& result0) { __builtin_mma_disassemble_acc(&result0.packet, c0); - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { - EIGEN_IF_CONSTEXPR(ConjugateLhs) { + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR (ConjugateLhs) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v; - } - else { - EIGEN_IF_CONSTEXPR(ConjugateRhs) { result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; } - else { + } else { + EIGEN_IF_CONSTEXPR (ConjugateRhs) { + result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; + } else { result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v; } } result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); - } - else EIGEN_IF_CONSTEXPR(sizeof(LhsPacket) == sizeof(std::complex)) { - EIGEN_IF_CONSTEXPR(ConjugateLhs) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; } - } - else { + } else EIGEN_IF_CONSTEXPR (sizeof(LhsPacket) == sizeof(std::complex)) { + EIGEN_IF_CONSTEXPR (ConjugateLhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + } + } else { result0.packet[0] = vec_mergee(result0.packet[0], result0.packet[1]); } } @@ -1777,10 +1798,9 @@ EIGEN_ALWAYS_INLINE void disassembleResults4(__vector_quad* c0, PacketBlock EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock& result0) { - EIGEN_IF_CONSTEXPR(!GEMV_IS_COMPLEX_FLOAT) { + EIGEN_IF_CONSTEXPR (!GEMV_IS_COMPLEX_FLOAT) { disassembleResults2(c0, result0); - } - else { + } else { disassembleResults4(c0, result0); } } @@ -1794,8 +1814,10 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter) { __builtin_mma_xxsetaccz(&e0##iter); } +#define GEMV_INIT_COL_COMPLEX_MMA(iter, N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \ + __builtin_mma_xxsetaccz(&e0##iter); \ + } #if EIGEN_COMP_LLVM #define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \ @@ -1804,39 +1826,35 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock(const_cast(&src)); \ EIGEN_UNUSED_VARIABLE(f##iter1); \ - } \ - else { \ + } else { \ f##iter1 = lhs.template load(i + ((iter2) * ResPacketSize), j); \ GEMV_BUILDPAIR_MMA(a##iter1, vec_splat(convertReal(f##iter1), 0), vec_splat(convertReal(f##iter1), 1)); \ } #endif #define GEMV_LOAD1_COL_COMPLEX_MMA(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \ f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \ EIGEN_UNUSED_VARIABLE(a##iter); \ - } \ - else { \ + } else { \ GEMV_LOADPAIR_COL_COMPLEX_MMA(iter, iter << 1) \ } \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(a##iter); \ EIGEN_UNUSED_VARIABLE(f##iter); \ } #define GEMV_WORK1_COL_COMPLEX_MMA(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \ gemv_mult_complex_MMA(f##iter, b, &e0##iter); \ - } \ - else { \ + } else { \ gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ } \ @@ -1846,17 +1864,15 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter1) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \ GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2); \ EIGEN_UNUSED_VARIABLE(a##iter3); \ - } \ - else { \ + } else { \ GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2 << 1); \ GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter3, iter3 << 1); \ } \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(a##iter2); \ EIGEN_UNUSED_VARIABLE(a##iter3); \ } \ @@ -1864,16 +1880,15 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter1) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \ PLhsPacket g[2]; \ __builtin_vsx_disassemble_pair(reinterpret_cast(g), &a##iter2); \ gemv_mult_complex_MMA(g[0], b, &e0##iter2); \ gemv_mult_complex_MMA(g[1], b, &e0##iter3); \ - } \ - else { \ + } else { \ gemv_mult_complex_MMA(a##iter2, b, &e0##iter2); \ gemv_mult_complex_MMA 1) { GEMV_UNROLL_HALF(GEMV_LOAD2_COL_COMPLEX_MMA, (N >> 1)) } \ - else { \ - GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) \ +#define GEMV_LOAD_COL_COMPLEX_MMA(N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_LOAD2_COL_COMPLEX_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) \ } -#define GEMV_WORK_COL_COMPLEX_MMA(N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > 1) { GEMV_UNROLL_HALF(GEMV_WORK2_COL_COMPLEX_MMA, (N >> 1)) } \ - else { \ - GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) \ +#define GEMV_WORK_COL_COMPLEX_MMA(N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_WORK2_COL_COMPLEX_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) \ } #else #define GEMV_LOAD_COL_COMPLEX_MMA(N) GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) @@ -1904,14 +1921,13 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \ GEMV_DISASSEMBLE_COMPLEX_MMA(iter); \ c0##iter = PResPacket(result0##iter.packet[0]); \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \ pstoreu_pmadd_complex( \ c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \ - } \ - else { \ + } else { \ pstoreu_pmadd_complex( \ c0##iter, alpha_data, res + i + ((iter << 1) * ResPacketSize)); \ c0##iter = PResPacket(result0##iter.packet[2]); \ @@ -1921,16 +1937,15 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter1) { \ GEMV_DISASSEMBLE_COMPLEX_MMA(iter2); \ GEMV_DISASSEMBLE_COMPLEX_MMA(iter3); \ c0##iter2 = PResPacket(result0##iter2.packet[0]); \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \ c0##iter3 = PResPacket(result0##iter3.packet[0]); \ pstoreu_pmadd_complex( \ c0##iter2, c0##iter3, alpha_data, res + i); \ - } \ - else { \ + } else { \ c0##iter3 = PResPacket(result0##iter2.packet[2]); \ pstoreu_pmadd_complex( \ c0##iter2, c0##iter3, alpha_data, res + i); \ @@ -1941,43 +1956,47 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock(&b1); \ - GEMV_UNROLL(GEMV_PREFETCH, N) \ - GEMV_LOAD_COL_COMPLEX_MMA(N) \ - GEMV_WORK_COL_COMPLEX_MMA(N) \ - } while (++j < jend); \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) <= 2) { GEMV_UNROLL(GEMV_STORE_COL_COMPLEX_MMA, N) } \ - else {GEMV_UNROLL_HALF(GEMV_STORE2_COL_COMPLEX_MMA, (N >> 1))} i += (ResPacketSize * N); +#define GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \ + GEMV_UNROLL(GEMV_INIT_COL_COMPLEX_MMA, N) \ + Index j = j2; \ + do { \ + const RhsScalar& b1 = rhs2(j, 0); \ + RhsScalar* b = const_cast(&b1); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_LOAD_COL_COMPLEX_MMA(N) \ + GEMV_WORK_COL_COMPLEX_MMA(N) \ + } while (++j < jend); \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) <= 2) { \ + GEMV_UNROLL(GEMV_STORE_COL_COMPLEX_MMA, N) \ + } else { \ + GEMV_UNROLL_HALF(GEMV_STORE2_COL_COMPLEX_MMA, (N >> 1)) \ + } \ + i += (ResPacketSize * N); #endif #define GEMV_INIT_COMPLEX(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { \ + EIGEN_IF_CONSTEXPR (N > iter) { \ c0##iter = pset_zero(); \ c1##iter = pset_init(c1##iter); \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(c0##iter); \ EIGEN_UNUSED_VARIABLE(c1##iter); \ } #define GEMV_WORK_COL_COMPLEX(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { \ + EIGEN_IF_CONSTEXPR (N > iter) { \ f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \ gemv_mult_complex(f##iter, b, c0##iter, c1##iter); \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(f##iter); \ } #define GEMV_STORE_COL_COMPLEX(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { c0##iter = padd(c0##iter, c1##iter); } \ + EIGEN_IF_CONSTEXPR (N > iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { \ + c0##iter = padd(c0##iter, c1##iter); \ + } \ pstoreu_pmadd_complex( \ c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \ } @@ -2003,10 +2022,11 @@ EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock 10) -#define GEMV_PROCESS_COL_COMPLEX(N) \ - EIGEN_IF_CONSTEXPR(sizeof(Scalar) != sizeof(LhsPacket)) { GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) } \ - else { \ - GEMV_PROCESS_COL_COMPLEX_ONE(N) \ +#define GEMV_PROCESS_COL_COMPLEX(N) \ + EIGEN_IF_CONSTEXPR (sizeof(Scalar) != sizeof(LhsPacket)) { \ + GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \ + } else { \ + GEMV_PROCESS_COL_COMPLEX_ONE(N) \ } #else #define GEMV_PROCESS_COL_COMPLEX(N) GEMV_PROCESS_COL_COMPLEX_ONE(N) @@ -2079,7 +2099,7 @@ EIGEN_STRONG_INLINE void gemv_complex_col(Index rows, Index cols, const LhsMappe GEMV_UNUSED(8, a) GEMV_UNUSED(8, f) #if !defined(GCC_ONE_VECTORPAIR_BUG) && defined(USE_GEMV_COL_COMPLEX_MMA) - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX || !GEMV_IS_COMPLEX_FLOAT) + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX || !GEMV_IS_COMPLEX_FLOAT) #endif #endif #ifndef GCC_ONE_VECTORPAIR_BUG @@ -2156,26 +2176,23 @@ EIGEN_ALWAYS_INLINE ScalarBlock, 2> addComplexResults(Packet result0.packet[2] = reinterpret_cast( vec_mergel(reinterpret_cast(result0.packet[2]), reinterpret_cast(result1.packet[2]))); result0.packet[0] = vec_add(result0.packet[0], result0.packet[2]); - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { result0.packet[1] = reinterpret_cast( vec_mergeh(reinterpret_cast(result0.packet[1]), reinterpret_cast(result1.packet[1]))); result0.packet[3] = reinterpret_cast( vec_mergel(reinterpret_cast(result0.packet[3]), reinterpret_cast(result1.packet[3]))); result0.packet[1] = vec_add(result0.packet[1], result0.packet[3]); - EIGEN_IF_CONSTEXPR(ConjugateLhs) { + EIGEN_IF_CONSTEXPR (ConjugateLhs) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v; - } - else EIGEN_IF_CONSTEXPR(ConjugateRhs) { + } else EIGEN_IF_CONSTEXPR (ConjugateRhs) { result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; - } - else { + } else { result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v; } result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); - } - else { - EIGEN_IF_CONSTEXPR(ConjugateLhs && (sizeof(LhsPacket) == sizeof(std::complex))) { + } else { + EIGEN_IF_CONSTEXPR (ConjugateLhs && (sizeof(LhsPacket) == sizeof(std::complex))) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; } } @@ -2219,23 +2236,20 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(__vector_quad* acc0 ScalarBlock cc0; PacketBlock result0; __builtin_mma_disassemble_acc(&result0.packet, acc0); - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { - EIGEN_IF_CONSTEXPR(ConjugateLhs) { + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR (ConjugateLhs) { result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v; result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v; - } - else EIGEN_IF_CONSTEXPR(ConjugateRhs) { + } else EIGEN_IF_CONSTEXPR (ConjugateRhs) { result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v; - } - else { + } else { result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v; result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v; } result0.packet[0] = vec_add(result0.packet[0], __builtin_vsx_xxpermdi(result0.packet[1], result0.packet[1], 2)); result0.packet[2] = vec_add(result0.packet[2], __builtin_vsx_xxpermdi(result0.packet[3], result0.packet[3], 2)); - } - else { + } else { result0.packet[0] = __builtin_vsx_xxpermdi(result0.packet[0], result0.packet[1], 1); result0.packet[2] = __builtin_vsx_xxpermdi(result0.packet[2], result0.packet[3], 1); } @@ -2273,59 +2287,64 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(ResPacket& a, ResPa #define GEMV_UNUSED_ROW(N, which) GEMV_UNROLL3_ROW(GEMV_UNUSED_VAR, N, which) -#define GEMV_INIT_ROW(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { __builtin_mma_xxsetaccz(&c##iter); } +#define GEMV_INIT_ROW(iter, N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \ + __builtin_mma_xxsetaccz(&c##iter); \ + } #define GEMV_LOADPAIR_ROW(iter1, iter2) \ GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_ROW(iter2), GEMV_LOADPACKET_ROW((iter2) + 1)); #define GEMV_WORK_ROW(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN(N) > iter) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ pger_vecMMA_acc(&c##iter, a0, GEMV_LOADPACKET_ROW(iter)); \ - } \ - else { \ + } else { \ __vector_pair b##iter; \ GEMV_LOADPAIR_ROW(iter, iter << 1) \ pger_vecMMA_acc(&c##iter, b##iter, a0); \ } \ } -#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(N > iter1) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_FLOAT) { cc##iter1 = predux_real(&c##iter2, &c##iter3); } \ - else { \ - cc##iter1 = predux_real(&c##iter1); \ - } \ - } \ - else { \ - EIGEN_UNUSED_VARIABLE(cc##iter1); \ +#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ + EIGEN_IF_CONSTEXPR (N > iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \ + cc##iter1 = predux_real(&c##iter2, &c##iter3); \ + } else { \ + cc##iter1 = predux_real(&c##iter1); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ } #else -#define GEMV_INIT_ROW(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { c##iter = pset1(ResScalar(0)); } \ - else { \ - EIGEN_UNUSED_VARIABLE(c##iter); \ +#define GEMV_INIT_ROW(iter, N) \ + EIGEN_IF_CONSTEXPR (N > iter) { \ + c##iter = pset1(ResScalar(0)); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c##iter); \ } -#define GEMV_WORK_ROW(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { c##iter = pcj.pmadd(GEMV_LOADPACKET_ROW(iter), a0, c##iter); } +#define GEMV_WORK_ROW(iter, N) \ + EIGEN_IF_CONSTEXPR (N > iter) { \ + c##iter = pcj.pmadd(GEMV_LOADPACKET_ROW(iter), a0, c##iter); \ + } -#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(N > iter1) { cc##iter1 = predux_real(c##iter2, c##iter3); } \ - else { \ - EIGEN_UNUSED_VARIABLE(cc##iter1); \ +#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ + EIGEN_IF_CONSTEXPR (N > iter1) { \ + cc##iter1 = predux_real(c##iter2, c##iter3); \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ } #endif #define GEMV_MULT(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(N > iter1) { \ + EIGEN_IF_CONSTEXPR (N > iter1) { \ cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), a0); \ cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), a0); \ } #define GEMV_STORE_ROW(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(N > iter1) { \ + EIGEN_IF_CONSTEXPR (N > iter1) { \ storeMaddData(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \ storeMaddData(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \ } @@ -2482,7 +2501,7 @@ EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16() template EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PResPacket& b0, ResPacket& a1, ResPacket& b1) { - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { a0 = padd(a0, a1); b0 = padd(b0, b1); } @@ -2509,20 +2528,21 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PRe GEMV_UNROLL_ROW_HALF(GEMV_STORE_ROW_COMPLEX, (N >> 1)) #ifdef USE_GEMV_MMA -#define GEMV_INIT_ROW_COMPLEX_MMA(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { __builtin_mma_xxsetaccz(&e0##iter); } +#define GEMV_INIT_ROW_COMPLEX_MMA(iter, N) \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \ + __builtin_mma_xxsetaccz(&e0##iter); \ + } #define GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter1, iter2) \ GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter2), GEMV_LOADPACKET_ROW_COMPLEX_DATA((iter2) + 1)); #define GEMV_WORK_ROW_COMPLEX_MMA(iter, N) \ - EIGEN_IF_CONSTEXPR(GEMV_GETN_COMPLEX(N) > iter) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \ PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \ gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ - } \ - else { \ + } else { \ __vector_pair a##iter; \ GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter, iter << 1) \ gemv_mult_complex_MMA predux_complex(PResPacket& a0, PRe } #define GEMV_PREDUX4_COMPLEX_MMA(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(N > iter1) { \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_FLOAT) { \ + EIGEN_IF_CONSTEXPR (N > iter1) { \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \ cc##iter1 = predux_complex( \ &e0##iter2, &e0##iter3); \ - } \ - else { \ + } else { \ cc##iter1 = \ predux_complex(&e0##iter1); \ } \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(cc##iter1); \ } @@ -2558,29 +2576,28 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PRe #endif #define GEMV_WORK_ROW_COMPLEX(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { \ + EIGEN_IF_CONSTEXPR (N > iter) { \ PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \ gemv_mult_complex(a##iter, b, c0##iter, c1##iter); \ } #define GEMV_PREDUX4_COMPLEX(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(N > iter1) { \ + EIGEN_IF_CONSTEXPR (N > iter1) { \ cc##iter1 = predux_complex(c0##iter2, c0##iter3, \ c1##iter2, c1##iter3); \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(cc##iter1); \ } #define GEMV_MULT_COMPLEX(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(N > iter1) { \ + EIGEN_IF_CONSTEXPR (N > iter1) { \ cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), b0); \ cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), b0); \ } #define GEMV_STORE_ROW_COMPLEX(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(N > iter1) { \ + EIGEN_IF_CONSTEXPR (N > iter1) { \ storeMaddData(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \ storeMaddData(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \ } @@ -2598,8 +2615,10 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PRe GEMV_PROCESS_END_ROW_COMPLEX(N); \ } -#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \ - EIGEN_IF_CONSTEXPR(GEMV_IS_COMPLEX_COMPLEX) { c0##iter = padd(c0##iter, c1##iter); } \ +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \ + EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { \ + c0##iter = padd(c0##iter, c1##iter); \ + } \ dd0 = predux(c0##iter); #if EIGEN_COMP_LLVM @@ -2614,25 +2633,25 @@ EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PRe #define GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter) lhs.template load(i + (iter), j) -#define GEMV_INIT_COMPLEX_OLD(iter, N) \ - EIGEN_UNUSED_VARIABLE(c0##iter); \ - EIGEN_IF_CONSTEXPR(N > iter) { c1##iter = pset_zero(); } \ - else { \ - EIGEN_UNUSED_VARIABLE(c1##iter); \ +#define GEMV_INIT_COMPLEX_OLD(iter, N) \ + EIGEN_UNUSED_VARIABLE(c0##iter); \ + EIGEN_IF_CONSTEXPR (N > iter) { \ + c1##iter = pset_zero(); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c1##iter); \ } #define GEMV_WORK_ROW_COMPLEX_OLD(iter, N) \ - EIGEN_IF_CONSTEXPR(N > iter) { \ + EIGEN_IF_CONSTEXPR (N > iter) { \ LhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter); \ c1##iter = pcj.pmadd(a##iter, b0, c1##iter); \ } #define GEMV_PREDUX4_COMPLEX_OLD(iter1, iter2, iter3, N) \ - EIGEN_IF_CONSTEXPR(N > iter1) { \ + EIGEN_IF_CONSTEXPR (N > iter1) { \ cc##iter1.scalar[0] = predux(c1##iter2); \ cc##iter1.scalar[1] = predux(c1##iter3); \ - } \ - else { \ + } else { \ EIGEN_UNUSED_VARIABLE(cc##iter1); \ } @@ -2743,11 +2762,15 @@ EIGEN_STRONG_INLINE void gemv_complex_row(Index rows, Index cols, const LhsMappe #ifndef GCC_ONE_VECTORPAIR_BUG ScalarBlock cc0, cc1, cc2, cc3; #ifdef USE_GEMV_MMA - EIGEN_IF_CONSTEXPR(!GEMV_IS_COMPLEX_COMPLEX) + EIGEN_IF_CONSTEXPR (!GEMV_IS_COMPLEX_COMPLEX) #endif - {GEMV_PROCESS_ROW_COMPLEX(8)} GEMV_PROCESS_ROW_COMPLEX(4) GEMV_PROCESS_ROW_COMPLEX(2) + { + GEMV_PROCESS_ROW_COMPLEX(8) + } + GEMV_PROCESS_ROW_COMPLEX(4) + GEMV_PROCESS_ROW_COMPLEX(2) #endif - for (; i < rows; ++i) { + for (; i < rows; ++i) { GEMV_PROCESS_ROW_COMPLEX_SINGLE(1) GEMV_PROCESS_ROW_COMPLEX_PREDUX(0) for (; j < cols; ++j) { diff --git a/Eigen/src/Core/arch/HVX/PacketMath.h b/Eigen/src/Core/arch/HVX/PacketMath.h index aaaee8437..0302db931 100644 --- a/Eigen/src/Core/arch/HVX/PacketMath.h +++ b/Eigen/src/Core/arch/HVX/PacketMath.h @@ -59,11 +59,10 @@ EIGEN_STRONG_INLINE HVX_Vector HVX_load_partial(const T* mem) { HVX_Vector v0 = HVX_vmem<0>(mem); HVX_Vector v1 = v0; uintptr_t mem_addr = reinterpret_cast(mem); - EIGEN_IF_CONSTEXPR(Size * sizeof(T) <= Alignment) { + EIGEN_IF_CONSTEXPR (Size * sizeof(T) <= Alignment) { // Data size less than alignment will never cross multiple aligned vectors. v1 = v0; - } - else { + } else { uintptr_t left_off = mem_addr & (__HVX_LENGTH__ - 1); if (left_off + Size * sizeof(T) > __HVX_LENGTH__) { v1 = HVX_vmem<1>(mem); @@ -99,7 +98,7 @@ EIGEN_STRONG_INLINE void HVX_store_partial(T* mem, HVX_Vector v) { HVX_VectorPred ql_not = Q6_Q_vsetq_R(mem_addr); HVX_VectorPred qr = Q6_Q_vsetq2_R(right_off); - EIGEN_IF_CONSTEXPR(Size * sizeof(T) > Alignment) { + EIGEN_IF_CONSTEXPR (Size * sizeof(T) > Alignment) { if (right_off > __HVX_LENGTH__) { Q6_vmem_QRIV(qr, mem + __HVX_LENGTH__ / sizeof(T), value); qr = Q6_Q_vcmp_eq_VbVb(value, value); diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h index d202ced60..f8533c486 100644 --- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h @@ -1144,13 +1144,14 @@ struct gebp_rhs_cols { static EIGEN_ALWAYS_INLINE void run(GEBPTraits& traits, const RhsScalar* blB, Index rhs_offset, LhsArray& A, RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) { constexpr int lane = J % 4; - EIGEN_IF_CONSTEXPR(lane == 0) - traits.loadRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel); - else traits.updateRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel); + EIGEN_IF_CONSTEXPR (lane == 0) + traits.loadRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel); + else + traits.updateRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel); - EIGEN_IF_CONSTEXPR(MrPackets >= 1) traits.madd(A[0], rhs_panel, C[J + 0 * NrCols], T0, fix); - EIGEN_IF_CONSTEXPR(MrPackets >= 2) traits.madd(A[1], rhs_panel, C[J + 1 * NrCols], T0, fix); - EIGEN_IF_CONSTEXPR(MrPackets >= 3) traits.madd(A[2], rhs_panel, C[J + 2 * NrCols], T0, fix); + EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.madd(A[0], rhs_panel, C[J + 0 * NrCols], T0, fix); + EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.madd(A[1], rhs_panel, C[J + 1 * NrCols], T0, fix); + EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.madd(A[2], rhs_panel, C[J + 2 * NrCols], T0, fix); gebp_rhs_cols::run(traits, blB, rhs_offset, A, rhs_panel, T0, C); } @@ -1166,9 +1167,9 @@ struct gebp_micro_step { RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) { constexpr int LhsProg = GEBPTraits::LhsProgress; - EIGEN_IF_CONSTEXPR(MrPackets >= 1) traits.loadLhs(&blA[(0 + MrPackets * K) * LhsProg], A[0]); - EIGEN_IF_CONSTEXPR(MrPackets >= 2) traits.loadLhs(&blA[(1 + MrPackets * K) * LhsProg], A[1]); - EIGEN_IF_CONSTEXPR(MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]); + EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.loadLhs(&blA[(0 + MrPackets * K) * LhsProg], A[0]); + EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.loadLhs(&blA[(1 + MrPackets * K) * LhsProg], A[1]); + EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]); gebp_rhs_cols<0, MrPackets, NrCols>::run(traits, blB, Index(NrCols * K), A, rhs_panel, T0, C); } @@ -1186,7 +1187,9 @@ EIGEN_ALWAYS_INLINE void gebp_neon_3p_workaround(LhsArray_& A) { #if EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0) using LhsElement = std::remove_all_extents_t>; constexpr bool apply = GEBPTraits_::Vectorizable && MrPackets == 3 && std::is_same::value; - EIGEN_IF_CONSTEXPR(apply) { __asm__("" : "+w,m"(A[0]), "+w,m"(A[1]), "+w,m"(A[2])); } + EIGEN_IF_CONSTEXPR (apply) { + __asm__("" : "+w,m"(A[0]), "+w,m"(A[1]), "+w,m"(A[2])); + } #else EIGEN_UNUSED_VARIABLE(A); #endif @@ -1204,7 +1207,7 @@ EIGEN_ALWAYS_INLINE void gebp_sse_spilling_workaround(LhsArray_& A, AccArray_& A using LhsElement = std::remove_all_extents_t>; constexpr bool apply = GEBPTraits_::Vectorizable && MrPackets <= 2 && NrCols >= 4 && std::is_same::value; - EIGEN_IF_CONSTEXPR(apply) { + EIGEN_IF_CONSTEXPR (apply) { #ifdef EIGEN_HAS_CXX17_IFCONSTEXPR using AccElement = std::decay_t; constexpr bool pin_acc = std::is_same::value && MrPackets == 2 && NrCols == 4; @@ -1214,7 +1217,9 @@ EIGEN_ALWAYS_INLINE void gebp_sse_spilling_workaround(LhsArray_& A, AccArray_& A "+x"(ACC[7])); } #else - EIGEN_IF_CONSTEXPR(MrPackets == 2) { __asm__("" : "+x,m"(A[0]), "+x,m"(A[1])); } + EIGEN_IF_CONSTEXPR (MrPackets == 2) { + __asm__("" : "+x,m"(A[0]), "+x,m"(A[1])); + } #endif } #endif @@ -1231,7 +1236,9 @@ struct gebp_peeled_loop { constexpr bool use_double_accum = (MrPackets == 1 && NrCols == 4); // Prefetch for 4-col paths - EIGEN_IF_CONSTEXPR(NrCols == 4) { internal::prefetch(blB + (48 + 0)); } + EIGEN_IF_CONSTEXPR (NrCols == 4) { + internal::prefetch(blB + (48 + 0)); + } // Helper to do one step with workarounds #define EIGEN_GEBP_DO_STEP(KVAL, ACC) \ @@ -1240,7 +1247,7 @@ struct gebp_peeled_loop { gebp_neon_3p_workaround(A); \ gebp_sse_spilling_workaround(A, ACC); \ /* LHS prefetch for 2pX4 and 3pX4 */ \ - EIGEN_IF_CONSTEXPR((MrPackets == 2 || MrPackets == 3) && NrCols == 4) { \ + EIGEN_IF_CONSTEXPR ((MrPackets == 2 || MrPackets == 3) && NrCols == 4) { \ internal::prefetch(blA + (MrPackets * KVAL + 16) * GEBPTraits::LhsProgress); \ if (EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) { \ internal::prefetch(blB + (NrCols * KVAL + 16) * GEBPTraits::RhsProgress); \ @@ -1248,23 +1255,26 @@ struct gebp_peeled_loop { } \ } while (false) - EIGEN_IF_CONSTEXPR(use_double_accum) { + EIGEN_IF_CONSTEXPR (use_double_accum) { EIGEN_GEBP_DO_STEP(0, C); EIGEN_GEBP_DO_STEP(1, D); EIGEN_GEBP_DO_STEP(2, C); EIGEN_GEBP_DO_STEP(3, D); - EIGEN_IF_CONSTEXPR(NrCols == 4) { internal::prefetch(blB + (48 + 16)); } + EIGEN_IF_CONSTEXPR (NrCols == 4) { + internal::prefetch(blB + (48 + 16)); + } EIGEN_GEBP_DO_STEP(4, C); EIGEN_GEBP_DO_STEP(5, D); EIGEN_GEBP_DO_STEP(6, C); EIGEN_GEBP_DO_STEP(7, D); - } - else { + } else { EIGEN_GEBP_DO_STEP(0, C); EIGEN_GEBP_DO_STEP(1, C); EIGEN_GEBP_DO_STEP(2, C); EIGEN_GEBP_DO_STEP(3, C); - EIGEN_IF_CONSTEXPR(NrCols == 4 && MrPackets == 2) { internal::prefetch(blB + (48 + 16)); } + EIGEN_IF_CONSTEXPR (NrCols == 4 && MrPackets == 2) { + internal::prefetch(blB + (48 + 16)); + } EIGEN_GEBP_DO_STEP(4, C); EIGEN_GEBP_DO_STEP(5, C); EIGEN_GEBP_DO_STEP(6, C); @@ -1321,7 +1331,7 @@ EIGEN_ALWAYS_INLINE void gebp_micro_panel_impl(GEBPTraits& traits, const DataMap // compiler that D[n] is always in bounds for the use_double_accum path. alignas(AccPacketLocal) AccPacketLocal D[CSize]; #endif - EIGEN_IF_CONSTEXPR(use_double_accum) { + EIGEN_IF_CONSTEXPR (use_double_accum) { for (int n = 0; n < NrCols; ++n) traits.initAcc(D[n]); } @@ -1353,7 +1363,7 @@ EIGEN_ALWAYS_INLINE void gebp_micro_panel_impl(GEBPTraits& traits, const DataMap } // Merge double accumulators - EIGEN_IF_CONSTEXPR(use_double_accum) { + EIGEN_IF_CONSTEXPR (use_double_accum) { for (int n = 0; n < NrCols; ++n) C[n] = padd(C[n], D[n]); } @@ -1456,7 +1466,7 @@ EIGEN_DONT_INLINE void gebp_kernel= 3 * Traits::LhsProgress) { + EIGEN_IF_CONSTEXPR (mr >= 3 * Traits::LhsProgress) { const Index rhs_block = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar); const Index lhs_strip = depth * sizeof(LhsScalar) * 3 * LhsProgress; const Index lhs_avail = (lhs_budget > rhs_block) ? (lhs_budget - rhs_block) : 0; @@ -1465,7 +1475,7 @@ EIGEN_DONT_INLINE void gebp_kernel(1, lhs_avail / lhs_strip); for (Index i1 = 0; i1 < peeled_mc3; i1 += actual_panel_rows) { const Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc3); - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { for (Index i = i1; i < actual_panel_end; i += 3 * LhsProgress) { micro_panel(fix<3>, fix<8>, traits, i, j2); @@ -1486,7 +1496,7 @@ EIGEN_DONT_INLINE void gebp_kernel= 2 * Traits::LhsProgress) { + EIGEN_IF_CONSTEXPR (mr >= 2 * Traits::LhsProgress) { const Index rhs_block2 = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar); const Index lhs_strip2 = depth * sizeof(LhsScalar) * 2 * LhsProgress; const Index lhs_avail2 = (lhs_budget > rhs_block2) ? (lhs_budget - rhs_block2) : 0; @@ -1496,7 +1506,7 @@ EIGEN_DONT_INLINE void gebp_kernel(1, lhs_avail2 / lhs_strip2); for (Index i1 = peeled_mc3; i1 < peeled_mc2; i1 += actual_panel_rows) { Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc2); - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { for (Index i = i1; i < actual_panel_end; i += 2 * LhsProgress) { micro_panel(fix<2>, fix<8>, traits, i, j2); @@ -1517,9 +1527,9 @@ EIGEN_DONT_INLINE void gebp_kernel= 1 * Traits::LhsProgress) { + EIGEN_IF_CONSTEXPR (mr >= 1 * Traits::LhsProgress) { for (Index i = peeled_mc2; i < peeled_mc1; i += LhsProgress) { - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { micro_panel(fix<1>, fix<8>, traits, i, j2); } @@ -1534,10 +1544,10 @@ EIGEN_DONT_INLINE void gebp_kernel= LhsProgressHalf) { + EIGEN_IF_CONSTEXPR ((LhsProgressHalf < LhsProgress) && mr >= LhsProgressHalf) { HalfTraits half_traits; for (Index i = peeled_mc1; i < peeled_mc_half; i += LhsProgressHalf) { - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { gebp_micro_panel_impl<1, 8, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper, LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB, @@ -1558,10 +1568,10 @@ EIGEN_DONT_INLINE void gebp_kernel= LhsProgressQuarter) { + EIGEN_IF_CONSTEXPR ((LhsProgressQuarter < LhsProgressHalf) && mr >= LhsProgressQuarter) { QuarterTraits quarter_traits; for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += LhsProgressQuarter) { - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { gebp_micro_panel_impl<1, 8, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper, LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB, @@ -1583,7 +1593,7 @@ EIGEN_DONT_INLINE void gebp_kernel= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { // loop on each panel of the rhs for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { // loop on each row of the lhs (1*LhsProgress x depth) @@ -1862,9 +1872,9 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= 3 * PacketSize) { + EIGEN_IF_CONSTEXPR (Pack1 >= 3 * PacketSize) { for (; i < peeled_mc3; i += 3 * PacketSize) { - EIGEN_IF_CONSTEXPR(PanelMode) count += (3 * PacketSize) * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += (3 * PacketSize) * offset; for (Index k = 0; k < depth; k++) { Packet A, B, C; @@ -1878,13 +1888,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= 2 * PacketSize) { + EIGEN_IF_CONSTEXPR (Pack1 >= 2 * PacketSize) { for (; i < peeled_mc2; i += 2 * PacketSize) { - EIGEN_IF_CONSTEXPR(PanelMode) count += (2 * PacketSize) * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += (2 * PacketSize) * offset; for (Index k = 0; k < depth; k++) { Packet A, B; @@ -1895,13 +1905,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= 1 * PacketSize) { + EIGEN_IF_CONSTEXPR (Pack1 >= 1 * PacketSize) { for (; i < peeled_mc1; i += 1 * PacketSize) { - EIGEN_IF_CONSTEXPR(PanelMode) count += (1 * PacketSize) * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += (1 * PacketSize) * offset; for (Index k = 0; k < depth; k++) { Packet A; @@ -1909,13 +1919,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= HalfPacketSize) { + EIGEN_IF_CONSTEXPR (HasHalf && Pack1 >= HalfPacketSize) { for (; i < peeled_mc_half; i += HalfPacketSize) { - EIGEN_IF_CONSTEXPR(PanelMode) count += (HalfPacketSize)*offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += (HalfPacketSize)*offset; for (Index k = 0; k < depth; k++) { HalfPacket A; @@ -1923,13 +1933,13 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= QuarterPacketSize) { + EIGEN_IF_CONSTEXPR (HasQuarter && Pack1 >= QuarterPacketSize) { for (; i < peeled_mc_quarter; i += QuarterPacketSize) { - EIGEN_IF_CONSTEXPR(PanelMode) count += (QuarterPacketSize)*offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += (QuarterPacketSize)*offset; for (Index k = 0; k < depth; k++) { QuarterPacket A; @@ -1937,7 +1947,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs 1) { + EIGEN_IF_CONSTEXPR (Pack2 < PacketSize && Pack2 > 1) { const Index pack2_progress = (HasHalf || HasQuarter) ? last_lhs_progress : Pack2; const Index peeled = (HasHalf || HasQuarter) ? peeled_mc0 : (rows / Pack2) * Pack2; for (; i < peeled; i += pack2_progress) { - EIGEN_IF_CONSTEXPR(PanelMode) count += pack2_progress * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * offset; for (Index k = 0; k < depth; k++) for (Index w = 0; w < pack2_progress; w++) blockA[count++] = cj(lhs(i + w, k)); - EIGEN_IF_CONSTEXPR(PanelMode) count += pack2_progress * (stride - offset - depth); + EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * (stride - offset - depth); } } // Pack scalars for (; i < rows; i++) { - EIGEN_IF_CONSTEXPR(PanelMode) count += offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += offset; for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k)); - EIGEN_IF_CONSTEXPR(PanelMode) count += (stride - offset - depth); + EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth); } } @@ -2012,7 +2022,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs 1 ? (rows / pack) * pack : 0 : i + (remaining_rows / pack) * pack; Index starting_pos = i; for (; i < peeled_mc; i += pack) { - EIGEN_IF_CONSTEXPR(PanelMode) count += pack * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += pack * offset; Index k = 0; if (pack >= psize && psize >= QuarterPacketSize) { @@ -2058,7 +2068,7 @@ EIGEN_DONT_INLINE void gemm_pack_lhs= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { // skip what we have before - EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset; const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); @@ -2142,10 +2152,10 @@ EIGEN_DONT_INLINE void gemm_pack_rhs kernel0, kernel1, kernel2, kernel3; kernel0.packet[0 % PacketSize] = dm0.template loadPacket(k); kernel0.packet[1 % PacketSize] = dm1.template loadPacket(k); @@ -2170,8 +2180,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs kernel0, kernel1; kernel0.packet[0 % PacketSize] = dm0.template loadPacket(k); @@ -2194,8 +2203,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs kernel0; kernel0.packet[0 % PacketSize] = dm0.template loadPacket(k); @@ -2233,28 +2241,28 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= 4) { + EIGEN_IF_CONSTEXPR (nr >= 4) { for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { // skip what we have before - EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset; const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3); Index k = 0; - EIGEN_IF_CONSTEXPR((PacketSize % 4) == 0 || PacketSize == 2) { + EIGEN_IF_CONSTEXPR ((PacketSize % 4) == 0 || PacketSize == 2) { for (; k < peeled_k; k += PacketSize) { PacketBlock kernel; kernel.packet[0] = dm0.template loadPacket(k); kernel.packet[1] = dm1.template loadPacket(k); kernel.packet[2] = dm2.template loadPacket(k); kernel.packet[3] = dm3.template loadPacket(k); - EIGEN_IF_CONSTEXPR(PacketSize == 2) { + EIGEN_IF_CONSTEXPR (PacketSize == 2) { // For PacketSize==2 we cannot ptranspose 4 packets directly; compose two // 2-packet transposes and re-interleave so the 4 stores produce the // packed-rhs layout (each store writing one half-row of the panel). @@ -2270,8 +2278,7 @@ EIGEN_DONT_INLINE void gemm_pack_rhs= 4 ? (cols / 4) * 4 : 0; Index count = 0; - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { // skip what we have before - EIGEN_IF_CONSTEXPR(PanelMode) count += 8 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset; for (Index k = 0; k < depth; k++) { - EIGEN_IF_CONSTEXPR(PacketSize == 8) { + EIGEN_IF_CONSTEXPR (PacketSize == 8) { Packet A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += PacketSize; - } - else EIGEN_IF_CONSTEXPR(PacketSize == 4) { + } else EIGEN_IF_CONSTEXPR (PacketSize == 4) { Packet A = rhs.template loadPacket(k, j2); Packet B = rhs.template loadPacket(k, j2 + 4); pstoreu(blockB + count, cj.pconj(A)); pstoreu(blockB + count + PacketSize, cj.pconj(B)); count += 2 * PacketSize; - } - else { + } else { const LinearMapper dm0 = rhs.getLinearMapper(k, j2); blockB[count + 0] = cj(dm0(0)); blockB[count + 1] = cj(dm0(1)); @@ -2361,31 +2366,28 @@ struct gemm_pack_rhs= 4) { + EIGEN_IF_CONSTEXPR (nr >= 4) { for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { // skip what we have before - EIGEN_IF_CONSTEXPR(PanelMode) count += 4 * offset; + EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset; for (Index k = 0; k < depth; k++) { - EIGEN_IF_CONSTEXPR(PacketSize == 4) { + EIGEN_IF_CONSTEXPR (PacketSize == 4) { Packet A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += PacketSize; - } - else EIGEN_IF_CONSTEXPR(HasHalf && HalfPacketSize == 4) { + } else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 4) { HalfPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += HalfPacketSize; - } - else EIGEN_IF_CONSTEXPR(HasQuarter && QuarterPacketSize == 4) { + } else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 4) { QuarterPacket A = rhs.template loadPacket(k, j2); pstoreu(blockB + count, cj.pconj(A)); count += QuarterPacketSize; - } - else { + } else { const LinearMapper dm0 = rhs.getLinearMapper(k, j2); blockB[count + 0] = cj(dm0(0)); blockB[count + 1] = cj(dm0(1)); @@ -2395,17 +2397,17 @@ struct gemm_pack_rhs processed with gebp or skipped // 2 - the actual_mc x actual_mc symmetric block => processed with a special kernel // 3 - after the diagonal => processed with gebp or skipped - EIGEN_IF_CONSTEXPR(UpLo == Lower) { + EIGEN_IF_CONSTEXPR (UpLo == Lower) { gebp(res.getSubMapper(i2, 0), blockA, blockB, actual_mc, actual_kc, (std::min)(size, i2), alpha, -1, -1, 0, 0); } @@ -129,7 +129,7 @@ struct general_matrix_matrix_triangular_product(BlockSize, size - j); const RhsScalar* actual_b = blockB + j * depth; - EIGEN_IF_CONSTEXPR(UpLo == Upper) { + EIGEN_IF_CONSTEXPR (UpLo == Upper) { gebp_kernel1(res.getSubMapper(0, j), blockA, actual_b, j, depth, actualBlockSize, alpha, -1, -1, 0, 0); } @@ -191,7 +191,7 @@ struct tribb_kernel { } } - EIGEN_IF_CONSTEXPR(UpLo == Lower) { + EIGEN_IF_CONSTEXPR (UpLo == Lower) { Index i = j + actualBlockSize; gebp_kernel1(res.getSubMapper(i, j), blockA + depth * i, actual_b, size - i, depth, actualBlockSize, alpha, -1, -1, 0, 0); @@ -240,7 +240,7 @@ struct general_product_to_triangular_selector(actualLhs.data()) : static_lhs.data())); - EIGEN_IF_CONSTEXPR(!UseLhsDirectly) { + EIGEN_IF_CONSTEXPR (!UseLhsDirectly) { Map(actualLhsPtr, actualLhs.size()) = actualLhs; } @@ -249,7 +249,7 @@ struct general_product_to_triangular_selector(actualRhs.data()) : static_rhs.data())); - EIGEN_IF_CONSTEXPR(!UseRhsDirectly) { + EIGEN_IF_CONSTEXPR (!UseRhsDirectly) { Map(actualRhsPtr, actualRhs.size()) = actualRhs; } diff --git a/Eigen/src/Core/products/GeneralMatrixVector.h b/Eigen/src/Core/products/GeneralMatrixVector.h index eb667a845..d7fddbaab 100644 --- a/Eigen/src/Core/products/GeneralMatrixVector.h +++ b/Eigen/src/Core/products/GeneralMatrixVector.h @@ -255,7 +255,7 @@ general_matrix_vector_product(i, j2, jend, lhs, rhs, res, palpha, pcj); i += ResPacketSize; } - EIGEN_IF_CONSTEXPR(HasHalf) { + EIGEN_IF_CONSTEXPR (HasHalf) { if (i < n_half) { ResPacketHalf c0 = pzero(ResPacketHalf{}); for (Index j = j2; j < jend; j += 1) { @@ -267,7 +267,7 @@ general_matrix_vector_product(i, j), b0, c0); } ResScalar cc0 = predux(c0); - EIGEN_IF_CONSTEXPR(HasHalf) { + EIGEN_IF_CONSTEXPR (HasHalf) { for (Index j = fullColBlockEnd; j < halfColBlockEnd; j += LhsPacketSizeHalf) { RhsPacketHalf b0 = rhs.template load(j, 0); c0_h = pcj_half.pmadd(lhs.template load(i, j), b0, c0_h); } cc0 += predux(c0_h); } - EIGEN_IF_CONSTEXPR(HasQuarter) { + EIGEN_IF_CONSTEXPR (HasQuarter) { for (Index j = halfColBlockEnd; j < quarterColBlockEnd; j += LhsPacketSizeQuarter) { RhsPacketQuarter b0 = rhs.template load(j, 0); c0_q = pcj_quarter.pmadd(lhs.template load(i, j), b0, c0_q); @@ -631,7 +631,7 @@ general_matrix_vector_product; ResScalar cc[N] = {}; - EIGEN_IF_CONSTEXPR(HasHalf) { + EIGEN_IF_CONSTEXPR (HasHalf) { ResPacketHalf h[N]; Unroll::init_zero(h); for (Index j = 0; j < halfColBlockEnd; j += LhsPacketSizeHalf) { @@ -640,7 +640,7 @@ general_matrix_vector_product= 3 * PacketSize) - for (Index i = 0; i < peeled_mc3; i += 3 * PacketSize) pack<3 * PacketSize>(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR (Pack1 >= 3 * PacketSize) + for (Index i = 0; i < peeled_mc3; i += 3 * PacketSize) pack<3 * PacketSize>(blockA, lhs, cols, i, count); - EIGEN_IF_CONSTEXPR(Pack1 >= 2 * PacketSize) - for (Index i = peeled_mc3; i < peeled_mc2; i += 2 * PacketSize) pack<2 * PacketSize>(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR (Pack1 >= 2 * PacketSize) + for (Index i = peeled_mc3; i < peeled_mc2; i += 2 * PacketSize) pack<2 * PacketSize>(blockA, lhs, cols, i, count); - EIGEN_IF_CONSTEXPR(Pack1 >= 1 * PacketSize) - for (Index i = peeled_mc2; i < peeled_mc1; i += 1 * PacketSize) pack<1 * PacketSize>(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR (Pack1 >= 1 * PacketSize) + for (Index i = peeled_mc2; i < peeled_mc1; i += 1 * PacketSize) pack<1 * PacketSize>(blockA, lhs, cols, i, count); - EIGEN_IF_CONSTEXPR(HasHalf && Pack1 >= HalfPacketSize) - for (Index i = peeled_mc1; i < peeled_mc_half; i += HalfPacketSize) - pack(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR (HasHalf && Pack1 >= HalfPacketSize) + for (Index i = peeled_mc1; i < peeled_mc_half; i += HalfPacketSize) + pack(blockA, lhs, cols, i, count); - EIGEN_IF_CONSTEXPR(HasQuarter && Pack1 >= QuarterPacketSize) - for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += QuarterPacketSize) - pack(blockA, lhs, cols, i, count); + EIGEN_IF_CONSTEXPR (HasQuarter && Pack1 >= QuarterPacketSize) + for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += QuarterPacketSize) + pack(blockA, lhs, cols, i, count); // do the same with mr==1 for (Index i = peeled_mc_quarter; i < rows; i++) { @@ -112,11 +112,11 @@ struct symm_pack_rhs { for (Index k = k2; k < end_k; k++) { blockB[count + 0] = rhs(k, j2 + 0); blockB[count + 1] = rhs(k, j2 + 1); - EIGEN_IF_CONSTEXPR(nr >= 4) { + EIGEN_IF_CONSTEXPR (nr >= 4) { blockB[count + 2] = rhs(k, j2 + 2); blockB[count + 3] = rhs(k, j2 + 3); } - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { blockB[count + 4] = rhs(k, j2 + 4); blockB[count + 5] = rhs(k, j2 + 5); blockB[count + 6] = rhs(k, j2 + 6); @@ -128,7 +128,7 @@ struct symm_pack_rhs { // second part: diagonal block Index end8 = nr >= 8 ? (std::min)(k2 + rows, packet_cols8) : k2; - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = k2; j2 < end8; j2 += 8) { // again we can split vertically in three different parts (transpose, symmetric, normal) // transpose @@ -170,7 +170,7 @@ struct symm_pack_rhs { } } } - EIGEN_IF_CONSTEXPR(nr >= 4) { + EIGEN_IF_CONSTEXPR (nr >= 4) { for (Index j2 = end8; j2 < (std::min)(k2 + rows, packet_cols4); j2 += 4) { // again we can split vertically in three different parts (transpose, symmetric, normal) // transpose @@ -206,7 +206,7 @@ struct symm_pack_rhs { } // third part: transposed - EIGEN_IF_CONSTEXPR(nr >= 8) { + EIGEN_IF_CONSTEXPR (nr >= 8) { for (Index j2 = k2 + rows; j2 < packet_cols8; j2 += 8) { for (Index k = k2; k < end_k; k++) { blockB[count + 0] = numext::conj(rhs(j2 + 0, k)); @@ -221,7 +221,7 @@ struct symm_pack_rhs { } } } - EIGEN_IF_CONSTEXPR(nr >= 4) { + EIGEN_IF_CONSTEXPR (nr >= 4) { for (Index j2 = (std::max)(packet_cols8, k2 + rows); j2 < packet_cols4; j2 += 4) { for (Index k = k2; k < end_k; k++) { blockB[count + 0] = numext::conj(rhs(j2 + 0, k)); diff --git a/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h b/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h index e5838bf7d..17df64f10 100644 --- a/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h +++ b/Eigen/src/Core/products/SelfadjointMatrixVector_BLAS.h @@ -57,11 +57,10 @@ struct selfadjoint_matrix_vector_product_symv Specialized> { \ static void run(Index size, const Scalar* lhs, Index lhsStride, const Scalar* _rhs, Scalar* res, Scalar alpha) { \ enum { IsColMajor = StorageOrder == ColMajor }; \ - EIGEN_IF_CONSTEXPR(IsColMajor == ConjugateLhs) { \ + EIGEN_IF_CONSTEXPR (IsColMajor == ConjugateLhs) { \ selfadjoint_matrix_vector_product::run(size, lhs, lhsStride, _rhs, res, alpha); \ - } \ - else { \ + } else { \ selfadjoint_matrix_vector_product_symv::run( \ size, lhs, lhsStride, _rhs, res, alpha); \ } \ diff --git a/Eigen/src/Core/products/SelfadjointProduct.h b/Eigen/src/Core/products/SelfadjointProduct.h index d486129f4..3cec348b5 100644 --- a/Eigen/src/Core/products/SelfadjointProduct.h +++ b/Eigen/src/Core/products/SelfadjointProduct.h @@ -40,7 +40,7 @@ struct selfadjoint_rank1_update Packet ps0 = internal::pset1(s0); Packet ps1 = internal::pset1(s1); - EIGEN_IF_CONSTEXPR(UpLo == Lower) { + EIGEN_IF_CONSTEXPR (UpLo == Lower) { Scalar* EIGEN_RESTRICT col0 = mat + stride * j + j; Scalar* EIGEN_RESTRICT col1 = mat + stride * (j + 1) + (j + 1); @@ -71,8 +71,7 @@ struct selfadjoint_rank1_update d0[k] += s0 * cx; d1[k] += s1 * cx; } - } - else { + } else { // UpLo == Upper Scalar* EIGEN_RESTRICT col0 = mat + stride * j; Scalar* EIGEN_RESTRICT col1 = mat + stride * (j + 1); @@ -162,7 +161,7 @@ struct selfadjoint_product_selector { Scalar, actualOtherPtr, other.size(), (UseOtherDirectly ? const_cast(actualOther.data()) : static_other.data())); - EIGEN_IF_CONSTEXPR(!UseOtherDirectly) { + EIGEN_IF_CONSTEXPR (!UseOtherDirectly) { Map(actualOtherPtr, actualOther.size()) = actualOther; } diff --git a/Eigen/src/Core/products/SelfadjointRank2Update.h b/Eigen/src/Core/products/SelfadjointRank2Update.h index a80d2f227..301b4cde2 100644 --- a/Eigen/src/Core/products/SelfadjointRank2Update.h +++ b/Eigen/src/Core/products/SelfadjointRank2Update.h @@ -220,7 +220,9 @@ EIGEN_DEVICE_FUNC SelfAdjointView& SelfAdjointView& SelfAdjointView(actualU.data()) : static_u.data())); - EIGEN_IF_CONSTEXPR(!UseUDirectly) { - EIGEN_IF_CONSTEXPR(NeedConjU) { Map(uPtr, size) = actualU.conjugate(); } - else { + EIGEN_IF_CONSTEXPR (!UseUDirectly) { + EIGEN_IF_CONSTEXPR (NeedConjU) { + Map(uPtr, size) = actualU.conjugate(); + } else { Map(uPtr, size) = actualU; } } @@ -241,9 +244,10 @@ EIGEN_DEVICE_FUNC SelfAdjointView& SelfAdjointView(actualV.data()) : static_v.data())); - EIGEN_IF_CONSTEXPR(!UseVDirectly) { - EIGEN_IF_CONSTEXPR(NeedConjV) { Map(vPtr, size) = actualV.conjugate(); } - else { + EIGEN_IF_CONSTEXPR (!UseVDirectly) { + EIGEN_IF_CONSTEXPR (NeedConjV) { + Map(vPtr, size) = actualV.conjugate(); + } else { Map(vPtr, size) = actualV; } } diff --git a/Eigen/src/Core/products/TriangularMatrixMatrix.h b/Eigen/src/Core/products/TriangularMatrixMatrix.h index 0a3b22b50..a6ec02516 100644 --- a/Eigen/src/Core/products/TriangularMatrixMatrix.h +++ b/Eigen/src/Core/products/TriangularMatrixMatrix.h @@ -92,9 +92,10 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix< Matrix triangularBuffer; triangularBuffer.setZero(); - EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag) - triangularBuffer.diagonal().setZero(); - else triangularBuffer.diagonal().setOnes(); + EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag) + triangularBuffer.diagonal().setZero(); + else + triangularBuffer.diagonal().setOnes(); gebp_kernel gebp_kernel; gemm_pack_lhs triangularBuffer; triangularBuffer.setZero(); - EIGEN_IF_CONSTEXPR((Mode & ZeroDiag) == ZeroDiag) - triangularBuffer.diagonal().setZero(); - else triangularBuffer.diagonal().setOnes(); + EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag) + triangularBuffer.diagonal().setZero(); + else + triangularBuffer.diagonal().setOnes(); gebp_kernel gebp_kernel; gemm_pack_lhs cols)) { actual_kc = cols - k2; k2 = actual_k2 + actual_kc - kc; @@ -355,7 +357,7 @@ struct triangular_product_impl { actualAlpha, blocking); // Apply correction if the diagonal is unit and a scalar factor was nested: - EIGEN_IF_CONSTEXPR((Mode & UnitDiag) == UnitDiag) { + EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) { if (LhsIsTriangular && !numext::is_exactly_one(lhs_alpha)) { Index diagSize = (std::min)(lhs.rows(), lhs.cols()); dst.topRows(diagSize) -= ((lhs_alpha - LhsScalar(1)) * a_rhs).topRows(diagSize); diff --git a/Eigen/src/Core/products/TriangularMatrixVector.h b/Eigen/src/Core/products/TriangularMatrixVector.h index 2791d5eb8..610d82533 100644 --- a/Eigen/src/Core/products/TriangularMatrixVector.h +++ b/Eigen/src/Core/products/TriangularMatrixVector.h @@ -55,7 +55,7 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product size) { general_matrix_vector_product::run( rows, cols - size, LhsMapper(&lhs_[size * lhsStride], lhsStride), RhsMapper(&rhs_[size * rhsIncr], rhsIncr), @@ -184,18 +183,17 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product diagSize) { general_matrix_vector_product::run( rows - diagSize, cols, LhsMapper(&lhs_[diagSize * lhsStride], lhsStride), RhsMapper(rhs_, rhsIncr), diff --git a/Eigen/src/Core/products/TriangularSolverMatrix.h b/Eigen/src/Core/products/TriangularSolverMatrix.h index eab16e839..6c631a438 100644 --- a/Eigen/src/Core/products/TriangularSolverMatrix.h +++ b/Eigen/src/Core/products/TriangularSolverMatrix.h @@ -98,7 +98,7 @@ EIGEN_STRONG_INLINE void trsmKernelR::Vectorizable) { + EIGEN_IF_CONSTEXPR (OtherInnerStride == 1 && packet_traits::Vectorizable) { using Packet = typename packet_traits::type; constexpr Index PS = unpacket_traits::size; // Unrolled k3 loop by 4 to reduce r load/store traffic. @@ -144,7 +144,7 @@ EIGEN_STRONG_INLINE void trsmKernelR(inv_rjj); Index i = 0; @@ -153,14 +153,13 @@ EIGEN_STRONG_INLINE void trsmKernelR::value || std::is_same::value))) { + EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 && + (std::is_same::value || std::is_same::value))) { // Very rough cutoffs to determine when to call trsm w/o packing // For small problem sizes trsmKernel compiled with clang is generally faster. // TODO: Investigate better heuristics for cutoffs. @@ -270,8 +269,8 @@ EIGEN_DONT_INLINE void triangular_solve_matrix::value || std::is_same::value))) { + EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 && + (std::is_same::value || std::is_same::value))) { i = IsLower ? k2 + k1 : k2 - k1 - actualPanelWidth; } #endif @@ -335,8 +334,8 @@ EIGEN_DONT_INLINE void triangular_solve_matrix::value || std::is_same::value))) { + EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 && + (std::is_same::value || std::is_same::value))) { // TODO: Investigate better heuristics for cutoffs. std::ptrdiff_t l1, l2, l3; manage_caching_sizes(GetAction, &l1, &l2, &l3); diff --git a/Eigen/src/Core/util/Memory.h b/Eigen/src/Core/util/Memory.h index 1ba8068c9..5b76fb9cf 100644 --- a/Eigen/src/Core/util/Memory.h +++ b/Eigen/src/Core/util/Memory.h @@ -494,7 +494,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) { if (size == 0) return nullptr; // short-cut. Also fixes Bug 884 check_size_for_overflow(size); T* result = static_cast(conditional_aligned_malloc(sizeof(T) * size)); - EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + EIGEN_IF_CONSTEXPR (NumTraits::RequireInitialization) { EIGEN_TRY { default_construct_elements_of_array(result, size); } EIGEN_CATCH(...) { conditional_aligned_free(result); @@ -506,7 +506,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) { template EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) { - EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + EIGEN_IF_CONSTEXPR (NumTraits::RequireInitialization) { return conditional_aligned_realloc_new(pts, new_size, old_size); } @@ -518,7 +518,7 @@ EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::si template EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T* ptr, std::size_t size) { - EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) destruct_elements_of_array(ptr, size); + EIGEN_IF_CONSTEXPR (NumTraits::RequireInitialization) destruct_elements_of_array(ptr, size); conditional_aligned_free(ptr); } @@ -673,12 +673,12 @@ class aligned_stack_memory_handler { **/ EIGEN_DEVICE_FUNC aligned_stack_memory_handler(T* ptr, std::size_t size, bool dealloc) : m_ptr(ptr), m_size(size), m_deallocate(dealloc) { - EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + EIGEN_IF_CONSTEXPR (NumTraits::RequireInitialization) { if (m_ptr) Eigen::internal::default_construct_elements_of_array(m_ptr, size); } } EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() { - EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + EIGEN_IF_CONSTEXPR (NumTraits::RequireInitialization) { if (m_ptr) Eigen::internal::destruct_elements_of_array(m_ptr, m_size); } if (m_deallocate) Eigen::internal::aligned_free(m_ptr); @@ -718,14 +718,14 @@ struct local_nested_eval_wrapper { : object(ptr == 0 ? reinterpret_cast(Eigen::internal::aligned_malloc(sizeof(Scalar) * xpr.size())) : ptr, xpr.rows(), xpr.cols()), m_deallocate(ptr == 0) { - EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + EIGEN_IF_CONSTEXPR (NumTraits::RequireInitialization) { if (object.data()) Eigen::internal::default_construct_elements_of_array(object.data(), object.size()); } object = xpr; } EIGEN_DEVICE_FUNC ~local_nested_eval_wrapper() { - EIGEN_IF_CONSTEXPR(NumTraits::RequireInitialization) { + EIGEN_IF_CONSTEXPR (NumTraits::RequireInitialization) { if (object.data()) Eigen::internal::destruct_elements_of_array(object.data(), object.size()); } if (m_deallocate) Eigen::internal::aligned_free(object.data()); diff --git a/Eigen/src/Geometry/OrthoMethods.h b/Eigen/src/Geometry/OrthoMethods.h index ed873b0a9..b8a9043d2 100644 --- a/Eigen/src/Geometry/OrthoMethods.h +++ b/Eigen/src/Geometry/OrthoMethods.h @@ -157,13 +157,12 @@ VectorwiseOp::cross(const MatrixBase& o typename internal::nested_eval::type vec(other.derived()); CrossReturnType res(_expression().rows(), _expression().cols()); - EIGEN_IF_CONSTEXPR(Direction == Vertical) { + EIGEN_IF_CONSTEXPR (Direction == Vertical) { eigen_assert(CrossReturnType::RowsAtCompileTime == 3 && "the matrix must have exactly 3 rows"); res.row(0) = (mat.row(1) * vec.coeff(2) - mat.row(2) * vec.coeff(1)).conjugate(); res.row(1) = (mat.row(2) * vec.coeff(0) - mat.row(0) * vec.coeff(2)).conjugate(); res.row(2) = (mat.row(0) * vec.coeff(1) - mat.row(1) * vec.coeff(0)).conjugate(); - } - else { + } else { eigen_assert(CrossReturnType::ColsAtCompileTime == 3 && "the matrix must have exactly 3 columns"); res.col(0) = (mat.col(1) * vec.coeff(2) - mat.col(2) * vec.coeff(1)).conjugate(); res.col(1) = (mat.col(2) * vec.coeff(0) - mat.col(0) * vec.coeff(2)).conjugate(); diff --git a/Eigen/src/Geometry/Transform.h b/Eigen/src/Geometry/Transform.h index 3dc563cb8..a46da6cba 100644 --- a/Eigen/src/Geometry/Transform.h +++ b/Eigen/src/Geometry/Transform.h @@ -1429,7 +1429,7 @@ struct transform_left_product_impl { typedef TransformType ResultType; static EIGEN_DEVICE_FUNC ResultType run(const Other& other, const TransformType& tr) { TransformType res; - EIGEN_IF_CONSTEXPR(Mode != int(AffineCompact)) res.matrix().row(Dim) = tr.matrix().row(Dim); + EIGEN_IF_CONSTEXPR (Mode != int(AffineCompact)) res.matrix().row(Dim) = tr.matrix().row(Dim); res.matrix().template topRows().noalias() = other * tr.matrix().template topRows(); return res; } diff --git a/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h b/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h index 7ed53c741..07a534125 100644 --- a/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h +++ b/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h @@ -142,7 +142,7 @@ class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner LeastSquareDiagonalPreconditioner& factorize(const MatType& mat) { // Compute the inverse squared-norm of each column of mat m_invdiag.resize(mat.cols()); - EIGEN_IF_CONSTEXPR(MatType::IsRowMajor) { + EIGEN_IF_CONSTEXPR (MatType::IsRowMajor) { m_invdiag.setZero(); for (Index j = 0; j < mat.outerSize(); ++j) { for (typename MatType::InnerIterator it(mat, j); it; ++it) m_invdiag(it.index()) += numext::abs2(it.value()); @@ -151,8 +151,7 @@ class LeastSquareDiagonalPreconditioner : public DiagonalPreconditioner RealScalar sum = numext::real(m_invdiag(j)); m_invdiag(j) = sum > RealScalar(0) ? RealScalar(1) / sum : RealScalar(1); } - } - else { + } else { for (Index j = 0; j < mat.outerSize(); ++j) { RealScalar sum = mat.col(j).squaredNorm(); m_invdiag(j) = sum > RealScalar(0) ? RealScalar(1) / sum : RealScalar(1); diff --git a/Eigen/src/SVD/JacobiSVD.h b/Eigen/src/SVD/JacobiSVD.h index 254039ec4..e7a998073 100644 --- a/Eigen/src/SVD/JacobiSVD.h +++ b/Eigen/src/SVD/JacobiSVD.h @@ -785,31 +785,29 @@ JacobiSVD& JacobiSVD::compute_impl(con // Non-blocking paths: apply rotations individually. The real and complex // paths are kept separate to avoid any codegen impact from the complex // preconditioner on GCC's optimization of the real inner loop. - else - EIGEN_IF_CONSTEXPR(NumTraits::IsComplex) { - // Complex non-blocking sweep: condition each 2x2 block to be real before diagonalizing. - for (Index p = 1; p < n; ++p) { - for (Index q = 0; q < p; ++q) { - RealScalar threshold = numext::maxi(considerAsZero, precision * maxDiagEntry); - if (abs(m_workMatrix.coeff(p, q)) > threshold || abs(m_workMatrix.coeff(q, p)) > threshold) { - finished = false; - if (internal::svd_precondition_2x2_block_to_be_real::run(m_workMatrix, *this, p, q, - maxDiagEntry)) { - JacobiRotation j_left, j_right; - internal::real_2x2_jacobi_svd(m_workMatrix, p, q, &j_left, &j_right); - m_workMatrix.applyOnTheLeft(p, q, j_left); - if (computeU()) m_matrixU.applyOnTheRight(p, q, j_left.transpose()); - m_workMatrix.applyOnTheRight(p, q, j_right); - if (computeV()) m_matrixV.applyOnTheRight(p, q, j_right); - maxDiagEntry = numext::maxi( - maxDiagEntry, - numext::maxi(abs(m_workMatrix.coeff(p, p)), abs(m_workMatrix.coeff(q, q)))); - } + else EIGEN_IF_CONSTEXPR (NumTraits::IsComplex) { + // Complex non-blocking sweep: condition each 2x2 block to be real before diagonalizing. + for (Index p = 1; p < n; ++p) { + for (Index q = 0; q < p; ++q) { + RealScalar threshold = numext::maxi(considerAsZero, precision * maxDiagEntry); + if (abs(m_workMatrix.coeff(p, q)) > threshold || abs(m_workMatrix.coeff(q, p)) > threshold) { + finished = false; + if (internal::svd_precondition_2x2_block_to_be_real::run(m_workMatrix, *this, p, q, + maxDiagEntry)) { + JacobiRotation j_left, j_right; + internal::real_2x2_jacobi_svd(m_workMatrix, p, q, &j_left, &j_right); + m_workMatrix.applyOnTheLeft(p, q, j_left); + if (computeU()) m_matrixU.applyOnTheRight(p, q, j_left.transpose()); + m_workMatrix.applyOnTheRight(p, q, j_right); + if (computeV()) m_matrixV.applyOnTheRight(p, q, j_right); + maxDiagEntry = numext::maxi( + maxDiagEntry, + numext::maxi(abs(m_workMatrix.coeff(p, p)), abs(m_workMatrix.coeff(q, q)))); } } } } - else { + } else { // Real non-blocking sweep: diagonalize each 2x2 block directly. RealScalar threshold = numext::maxi(considerAsZero, precision * maxDiagEntry); for (Index p = 1; p < n; ++p) { @@ -840,7 +838,7 @@ JacobiSVD& JacobiSVD::compute_impl(con // treated by svd_precondition_2x2_block_to_be_real, and the imaginary part // of some diagonal entry might not be null. bool diagonal_has_imaginary_part = false; - EIGEN_IF_CONSTEXPR(NumTraits::IsComplex) { + EIGEN_IF_CONSTEXPR (NumTraits::IsComplex) { diagonal_has_imaginary_part = abs(numext::imag(m_workMatrix.coeff(i, i))) > considerAsZero; } if (diagonal_has_imaginary_part) { @@ -952,7 +950,7 @@ EIGEN_DONT_INLINE bool JacobiSVD::blocked_sweep(RealScalar // Right operations (column scaling) are applied directly since column // ops are contiguous in column-major layout. bool doRealSvd = true; - EIGEN_IF_CONSTEXPR(NumTraits::IsComplex) { + EIGEN_IF_CONSTEXPR (NumTraits::IsComplex) { Scalar z; // nn = ||(w_pp, w_qp)||_2, the norm of the first column of the 2x2 block. RealScalar nn = sqrt(numext::abs2(blockBuffer.coeff(kBlockSize, kBlockSize)) + @@ -1066,7 +1064,7 @@ EIGEN_DONT_INLINE bool JacobiSVD::blocked_sweep(RealScalar notFinished = true; bool doRealSvd = true; - EIGEN_IF_CONSTEXPR(NumTraits::IsComplex) { + EIGEN_IF_CONSTEXPR (NumTraits::IsComplex) { doRealSvd = internal::svd_precondition_2x2_block_to_be_real::run(m_workMatrix, *this, p, q, maxDiagEntry); } diff --git a/Eigen/src/SVD/SVDBase.h b/Eigen/src/SVD/SVDBase.h index d150c0bd8..b8547364f 100644 --- a/Eigen/src/SVD/SVDBase.h +++ b/Eigen/src/SVD/SVDBase.h @@ -424,10 +424,10 @@ bool SVDBase::allocate(Index rows, Index cols, unsigned int computation m_diagSize.setValue(numext::mini(m_rows.value(), m_cols.value())); m_singularValues.resize(m_diagSize.value()); - EIGEN_IF_CONSTEXPR(RowsAtCompileTime == Dynamic) { + EIGEN_IF_CONSTEXPR (RowsAtCompileTime == Dynamic) { m_matrixU.resize(m_rows.value(), m_computeFullU ? m_rows.value() : m_computeThinU ? m_diagSize.value() : 0); } - EIGEN_IF_CONSTEXPR(ColsAtCompileTime == Dynamic) { + EIGEN_IF_CONSTEXPR (ColsAtCompileTime == Dynamic) { m_matrixV.resize(m_cols.value(), m_computeFullV ? m_cols.value() : m_computeThinV ? m_diagSize.value() : 0); } diff --git a/Eigen/src/SparseCholesky/SimplicialCholesky.h b/Eigen/src/SparseCholesky/SimplicialCholesky.h index 23fb80abc..33c08b2b1 100644 --- a/Eigen/src/SparseCholesky/SimplicialCholesky.h +++ b/Eigen/src/SparseCholesky/SimplicialCholesky.h @@ -867,7 +867,7 @@ void SimplicialCholeskyBase::ordering(const MatrixType& a, ConstCholMat const Index size = a.rows(); pmat = ≈ // Note that ordering methods compute the inverse permutation - EIGEN_IF_CONSTEXPR((!std::is_same >::value)) { + EIGEN_IF_CONSTEXPR ((!std::is_same >::value)) { { CholMatrixType C; constexpr bool kUseAMDFastPath = std::is_same >::value; @@ -882,16 +882,15 @@ void SimplicialCholeskyBase::ordering(const MatrixType& a, ConstCholMat ap.resize(size, size); internal::permute_symm_to_symm(a, ap, m_P.indices().data()); - } - else { + } else { m_Pinv.resize(0); m_P.resize(0); - EIGEN_IF_CONSTEXPR(int(UpLo) == int(Lower) || MatrixType::IsRowMajor) { + EIGEN_IF_CONSTEXPR (int(UpLo) == int(Lower) || MatrixType::IsRowMajor) { // we have to transpose the lower part to to the upper one ap.resize(size, size); internal::permute_symm_to_symm(a, ap, NULL); - } - else internal::simplicial_cholesky_grab_input::run(a, pmat, ap); + } else + internal::simplicial_cholesky_grab_input::run(a, pmat, ap); } } diff --git a/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h b/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h index c2d6a3ce7..dcdd99a19 100644 --- a/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h +++ b/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h @@ -376,8 +376,9 @@ void SimplicialCholeskyBase::factorize_preordered(const CholMatrixType& Index p = Lp[k] + nonZerosPerCol[k]++; Li[p] = k; /* store L(k,k) = sqrt (d) in column k */ bool failed; - EIGEN_IF_CONSTEXPR(NonHermitian) { failed = d == RealScalar(0); } - else { + EIGEN_IF_CONSTEXPR (NonHermitian) { + failed = d == RealScalar(0); + } else { failed = numext::real(d) <= RealScalar(0); } if (failed) { diff --git a/Eigen/src/SparseCore/SparseAssign.h b/Eigen/src/SparseCore/SparseAssign.h index 36f02b048..1d7a4401e 100644 --- a/Eigen/src/SparseCore/SparseAssign.h +++ b/Eigen/src/SparseCore/SparseAssign.h @@ -252,9 +252,9 @@ struct Assignment { template struct Assignment { static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) { - EIGEN_IF_CONSTEXPR( - (std::is_same>::value)) - dst.setZero(); + EIGEN_IF_CONSTEXPR ((std::is_same>::value)) + dst.setZero(); internal::evaluator srcEval(src); resize_if_allowed(dst, src, func); diff --git a/Eigen/src/SparseCore/SparseBlock.h b/Eigen/src/SparseCore/SparseBlock.h index c5e2bf0f3..4520491aa 100644 --- a/Eigen/src/SparseCore/SparseBlock.h +++ b/Eigen/src/SparseCore/SparseBlock.h @@ -204,11 +204,10 @@ class sparse_matrix_block_impl : public SparseCompressedBase(tmp.innerVector(k).nonZeros()); diff --git a/Eigen/src/SparseCore/SparseDenseProduct.h b/Eigen/src/SparseCore/SparseDenseProduct.h index 1c6de14b5..94e469557 100644 --- a/Eigen/src/SparseCore/SparseDenseProduct.h +++ b/Eigen/src/SparseCore/SparseDenseProduct.h @@ -198,7 +198,7 @@ struct sparse_time_dense_product_impl reports innerStride()==1 but is actually RowMajor, so check both. - EIGEN_IF_CONSTEXPR(!(Res::Flags & RowMajorBit)) { + EIGEN_IF_CONSTEXPR (!(Res::Flags & RowMajorBit)) { if (res.innerStride() == 1) { for (Index c = 0; c < rhs.cols(); ++c) { typename Res::Scalar* y = res.data() + c * res.outerStride(); diff --git a/Eigen/src/SparseCore/SparseDiagonalProduct.h b/Eigen/src/SparseCore/SparseDiagonalProduct.h index cee2aeca3..222b6ab02 100644 --- a/Eigen/src/SparseCore/SparseDiagonalProduct.h +++ b/Eigen/src/SparseCore/SparseDiagonalProduct.h @@ -144,8 +144,9 @@ struct sparse_selfadjoint_diagonal_product_impl { const StorageIndex outer = outerIndex(row, col); const Index k = count[outer]++; dest.innerIndexPtr()[k] = innerIndex(row, col); - EIGEN_IF_CONSTEXPR(ProductOrder == OnTheLeft) { dest.valuePtr()[k] = diagonal.coeff(row) * coeff; } - else { + EIGEN_IF_CONSTEXPR (ProductOrder == OnTheLeft) { + dest.valuePtr()[k] = diagonal.coeff(row) * coeff; + } else { dest.valuePtr()[k] = coeff * diagonal.coeff(col); } } diff --git a/Eigen/src/SparseCore/SparseMatrixBase.h b/Eigen/src/SparseCore/SparseMatrixBase.h index 7484f8bbc..edc19b3a0 100644 --- a/Eigen/src/SparseCore/SparseMatrixBase.h +++ b/Eigen/src/SparseCore/SparseMatrixBase.h @@ -228,7 +228,7 @@ class SparseMatrixBase : public EigenBase { using Nested = typename Derived::Nested; using NestedCleaned = internal::remove_all_t; - EIGEN_IF_CONSTEXPR(Flags & RowMajorBit) { + EIGEN_IF_CONSTEXPR (Flags & RowMajorBit) { Nested nm(m.derived()); internal::evaluator thisEval(nm); @@ -268,8 +268,7 @@ class SparseMatrixBase : public EigenBase { } s << std::endl; } - } - else { + } else { Nested nm(m.derived()); internal::evaluator thisEval(nm); if (m.cols() == 1) { diff --git a/Eigen/src/SparseCore/SparseRef.h b/Eigen/src/SparseCore/SparseRef.h index 995499357..f5a9916e5 100644 --- a/Eigen/src/SparseCore/SparseRef.h +++ b/Eigen/src/SparseCore/SparseRef.h @@ -90,18 +90,16 @@ class SparseRefBase : public SparseMapBase { protected: template void construct(Expression& expr) { - EIGEN_IF_CONSTEXPR(Expression::IsVectorAtCompileTime) { + EIGEN_IF_CONSTEXPR (Expression::IsVectorAtCompileTime) { const Index offset = expr.outerIndexPtr() ? expr.outerIndexPtr()[0] : 0; auto inner_index_ptr = expr.innerIndexPtr(); auto value_ptr = expr.valuePtr(); if (inner_index_ptr) inner_index_ptr += offset; if (value_ptr) value_ptr += offset; internal::construct_at(this, expr.size(), expr.nonZeros(), inner_index_ptr, value_ptr); - } - else if (expr.outerIndexPtr() == 0) { + } else if (expr.outerIndexPtr() == 0) { internal::construct_at(this, expr.size(), expr.nonZeros(), expr.innerIndexPtr(), expr.valuePtr()); - } - else { + } else { internal::construct_at(this, expr.rows(), expr.cols(), expr.nonZeros(), expr.outerIndexPtr(), expr.innerIndexPtr(), expr.valuePtr(), expr.innerNonZeroPtr()); } diff --git a/Eigen/src/SparseCore/SparseSelfAdjointView.h b/Eigen/src/SparseCore/SparseSelfAdjointView.h index bda6e91c7..a556595ec 100644 --- a/Eigen/src/SparseCore/SparseSelfAdjointView.h +++ b/Eigen/src/SparseCore/SparseSelfAdjointView.h @@ -335,7 +335,9 @@ inline void sparse_selfadjoint_time_dense_product(const SparseLhsType& lhs, cons typename DenseResType::Scalar res_j(0); for (; (ProcessFirstHalf ? i && i.index() < j : i); ++i) { LhsScalar lhs_ij = i.value(); - EIGEN_IF_CONSTEXPR(!LhsIsRowMajor) { lhs_ij = numext::conj(lhs_ij); } + EIGEN_IF_CONSTEXPR (!LhsIsRowMajor) { + lhs_ij = numext::conj(lhs_ij); + } res_j += lhs_ij * rhs.coeff(i.index(), k); res(i.index(), k) += numext::conj(lhs_ij) * rhs_j; } @@ -456,9 +458,10 @@ void permute_symm_to_fullsymm( Index r = it.row(); Index c = it.col(); Index ip = perm ? perm[i] : i; - EIGEN_IF_CONSTEXPR(Mode == int(Upper | Lower)) - count[StorageOrderMatch ? jp : ip]++; - else if (r == c) count[ip]++; + EIGEN_IF_CONSTEXPR (Mode == int(Upper | Lower)) + count[StorageOrderMatch ? jp : ip]++; + else if (r == c) + count[ip]++; else if ((Mode == Lower && r > c) || (Mode == Upper && r < c)) { count[ip]++; count[jp]++; @@ -483,18 +486,16 @@ void permute_symm_to_fullsymm( StorageIndex jp = perm ? perm[j] : j; StorageIndex ip = perm ? perm[i] : i; - EIGEN_IF_CONSTEXPR(Mode == int(Upper | Lower)) { + EIGEN_IF_CONSTEXPR (Mode == int(Upper | Lower)) { Index k = count[StorageOrderMatch ? jp : ip]++; dest.innerIndexPtr()[k] = StorageOrderMatch ? ip : jp; dest.valuePtr()[k] = it.value(); - } - else if (r == c) { + } else if (r == c) { Index k = count[ip]++; dest.innerIndexPtr()[k] = ip; dest.valuePtr()[k] = it.value(); - } - else if (((Mode & Lower) == Lower && r > c) || ((Mode & Upper) == Upper && r < c)) { - EIGEN_IF_CONSTEXPR(!StorageOrderMatch) std::swap(ip, jp); + } else if (((Mode & Lower) == Lower && r > c) || ((Mode & Upper) == Upper && r < c)) { + EIGEN_IF_CONSTEXPR (!StorageOrderMatch) std::swap(ip, jp); Index k = count[jp]++; dest.innerIndexPtr()[k] = ip; dest.valuePtr()[k] = it.value(); @@ -556,7 +557,7 @@ void permute_symm_to_symm(const MatrixType& mat, Index k = count[int(DstMode) == int(Lower) ? (std::min)(ip, jp) : (std::max)(ip, jp)]++; dest.innerIndexPtr()[k] = int(DstMode) == int(Lower) ? (std::max)(ip, jp) : (std::min)(ip, jp); - EIGEN_IF_CONSTEXPR(!StorageOrderMatch) std::swap(ip, jp); + EIGEN_IF_CONSTEXPR (!StorageOrderMatch) std::swap(ip, jp); if (((int(DstMode) == int(Lower) && ip < jp) || (int(DstMode) == int(Upper) && ip > jp))) dest.valuePtr()[k] = (NonHermitian ? it.value() : numext::conj(it.value())); else diff --git a/Eigen/src/SparseCore/SparseSparseProductWithPruning.h b/Eigen/src/SparseCore/SparseSparseProductWithPruning.h index bb4022bdb..9fa33efd9 100644 --- a/Eigen/src/SparseCore/SparseSparseProductWithPruning.h +++ b/Eigen/src/SparseCore/SparseSparseProductWithPruning.h @@ -38,8 +38,9 @@ static void sparse_sparse_product_with_pruning_impl(const Lhs& lhs, const Rhs& r AmbiVector tempVector(rows); // mimics a resizeByInnerOuter: - EIGEN_IF_CONSTEXPR(ResultType::IsRowMajor) { res.resize(cols, rows); } - else { + EIGEN_IF_CONSTEXPR (ResultType::IsRowMajor) { + res.resize(cols, rows); + } else { res.resize(rows, cols); } diff --git a/Eigen/src/SparseCore/SparseTriangularView.h b/Eigen/src/SparseCore/SparseTriangularView.h index b595b26cb..2bbcaa43b 100644 --- a/Eigen/src/SparseCore/SparseTriangularView.h +++ b/Eigen/src/SparseCore/SparseTriangularView.h @@ -105,12 +105,11 @@ struct unary_evaluator, IteratorBased> : evaluator : Base(xprEval.m_argImpl, outer), m_returnOne(false), m_containsDiag(Base::outer() < xprEval.m_arg.innerSize()) { - EIGEN_IF_CONSTEXPR(SkipFirst) { + EIGEN_IF_CONSTEXPR (SkipFirst) { while ((*this) && ((HasUnitDiag || SkipDiag) ? this->index() <= outer : this->index() < outer)) Base::operator++(); - EIGEN_IF_CONSTEXPR(HasUnitDiag) m_returnOne = m_containsDiag; - } - else EIGEN_IF_CONSTEXPR(HasUnitDiag) { + EIGEN_IF_CONSTEXPR (HasUnitDiag) m_returnOne = m_containsDiag; + } else EIGEN_IF_CONSTEXPR (HasUnitDiag) { if ((!Base::operator bool()) || Base::index() >= Base::outer()) { if (Base::operator bool()) Base::operator++(); m_returnOne = m_containsDiag; @@ -119,14 +118,14 @@ struct unary_evaluator, IteratorBased> : evaluator } EIGEN_STRONG_INLINE InnerIterator& operator++() { - EIGEN_IF_CONSTEXPR(HasUnitDiag) { + EIGEN_IF_CONSTEXPR (HasUnitDiag) { if (m_returnOne) { m_returnOne = false; return *this; } } Base::operator++(); - EIGEN_IF_CONSTEXPR(HasUnitDiag && !SkipFirst) { + EIGEN_IF_CONSTEXPR (HasUnitDiag && !SkipFirst) { if ((!Base::operator bool()) || Base::index() >= Base::outer()) { if (Base::operator bool()) Base::operator++(); m_returnOne = m_containsDiag; @@ -136,13 +135,15 @@ struct unary_evaluator, IteratorBased> : evaluator } EIGEN_STRONG_INLINE operator bool() const { - EIGEN_IF_CONSTEXPR(HasUnitDiag) { + EIGEN_IF_CONSTEXPR (HasUnitDiag) { if (m_returnOne) return true; } - EIGEN_IF_CONSTEXPR(SkipFirst) { return Base::operator bool(); } - else { - EIGEN_IF_CONSTEXPR(SkipDiag) { return (Base::operator bool() && this->index() < this->outer()); } - else { + EIGEN_IF_CONSTEXPR (SkipFirst) { + return Base::operator bool(); + } else { + EIGEN_IF_CONSTEXPR (SkipDiag) { + return (Base::operator bool() && this->index() < this->outer()); + } else { return (Base::operator bool() && this->index() <= this->outer()); } } @@ -151,13 +152,13 @@ struct unary_evaluator, IteratorBased> : evaluator inline Index row() const { return (ArgType::Flags & RowMajorBit ? Base::outer() : this->index()); } inline Index col() const { return (ArgType::Flags & RowMajorBit ? this->index() : Base::outer()); } inline StorageIndex index() const { - EIGEN_IF_CONSTEXPR(HasUnitDiag) { + EIGEN_IF_CONSTEXPR (HasUnitDiag) { if (m_returnOne) return internal::convert_index(Base::outer()); } return Base::index(); } inline Scalar value() const { - EIGEN_IF_CONSTEXPR(HasUnitDiag) { + EIGEN_IF_CONSTEXPR (HasUnitDiag) { if (m_returnOne) return Scalar(1); } return Base::value(); diff --git a/Eigen/src/SparseCore/SparsityPatternRef.h b/Eigen/src/SparseCore/SparsityPatternRef.h index d9684a765..0204c823b 100644 --- a/Eigen/src/SparseCore/SparsityPatternRef.h +++ b/Eigen/src/SparseCore/SparsityPatternRef.h @@ -382,8 +382,9 @@ void materialize_selfadjoint_pattern(const SparsityPatternRef& A, for (Index j = 0; j < n; ++j) { const StorageIndex* a_col = A.inner + A.outer[j]; const Index a_nz = A.nonZeros(j); - EIGEN_IF_CONSTEXPR(IsLower) { a_split(j) = std::lower_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col; } - else { + EIGEN_IF_CONSTEXPR (IsLower) { + a_split(j) = std::lower_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col; + } else { a_split(j) = std::upper_bound(a_col, a_col + a_nz, StorageIndex(j)) - a_col; } } diff --git a/Eigen/src/SparseCore/TriangularSolver.h b/Eigen/src/SparseCore/TriangularSolver.h index 2c8d0af0d..af95e4203 100644 --- a/Eigen/src/SparseCore/TriangularSolver.h +++ b/Eigen/src/SparseCore/TriangularSolver.h @@ -44,8 +44,8 @@ struct sparse_solve_triangular_selector { if (lastIndex == i) break; tmp = numext::madd(-lastVal, other.coeff(lastIndex, col), tmp); } - EIGEN_IF_CONSTEXPR(Mode & UnitDiag) - other.coeffRef(i, col) = tmp; + EIGEN_IF_CONSTEXPR (Mode & UnitDiag) + other.coeffRef(i, col) = tmp; else { eigen_assert(lastIndex == i); other.coeffRef(i, col) = tmp / lastVal; @@ -69,19 +69,20 @@ struct sparse_solve_triangular_selector { Scalar l_ii(0); LhsIterator it(lhsEval, i); while (it && it.index() < i) ++it; - EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) { + EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) { eigen_assert(it && it.index() == i); l_ii = it.value(); ++it; - } - else if (it && it.index() == i)++ it; + } else if (it && it.index() == i) + ++it; for (; it; ++it) { tmp = numext::madd(-it.value(), other.coeff(it.index(), col), tmp); } - EIGEN_IF_CONSTEXPR(Mode & UnitDiag) - other.coeffRef(i, col) = tmp; - else other.coeffRef(i, col) = tmp / l_ii; + EIGEN_IF_CONSTEXPR (Mode & UnitDiag) + other.coeffRef(i, col) = tmp; + else + other.coeffRef(i, col) = tmp / l_ii; } } } @@ -102,7 +103,7 @@ struct sparse_solve_triangular_selector { { LhsIterator it(lhsEval, i); while (it && it.index() < i) ++it; - EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) { + EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) { eigen_assert(it && it.index() == i); tmp /= it.value(); } @@ -129,7 +130,7 @@ struct sparse_solve_triangular_selector { Scalar& tmp = other.coeffRef(i, col); if (!numext::is_exactly_zero(tmp)) // optimization when other is actually sparse { - EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) { + EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) { // TODO: replace this with a binary search. make sure the binary search is safe for partially sorted // elements LhsIterator it(lhsEval, i); @@ -210,21 +211,20 @@ struct sparse_solve_triangular_sparse_selector { if (!numext::is_exactly_zero(ci)) { // find typename Lhs::InnerIterator it(lhs, i); - EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) { - EIGEN_IF_CONSTEXPR(IsLower) { + EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) { + EIGEN_IF_CONSTEXPR (IsLower) { eigen_assert(it.index() == i); ci /= it.value(); - } - else ci /= lhs.coeff(i, i); + } else + ci /= lhs.coeff(i, i); } tempVector.restart(); - EIGEN_IF_CONSTEXPR(IsLower) { + EIGEN_IF_CONSTEXPR (IsLower) { if (it.index() == i) ++it; for (; it; ++it) { tempVector.coeffRef(it.index()) = numext::madd(-ci, it.value(), tempVector.coeffRef(it.index())); } - } - else { + } else { for (; it && it.index() < i; ++it) { tempVector.coeffRef(it.index()) = numext::madd(-ci, it.value(), tempVector.coeffRef(it.index())); } diff --git a/Eigen/src/SuperLUSupport/SuperLUSupport.h b/Eigen/src/SuperLUSupport/SuperLUSupport.h index 3cc0b1b4e..9f98199cd 100644 --- a/Eigen/src/SuperLUSupport/SuperLUSupport.h +++ b/Eigen/src/SuperLUSupport/SuperLUSupport.h @@ -155,11 +155,14 @@ struct SluMatrix : SuperMatrix { template void setScalarType() { - EIGEN_IF_CONSTEXPR((std::is_same::value)) - Dtype = SLU_S; - else EIGEN_IF_CONSTEXPR((std::is_same::value)) Dtype = SLU_D; - else EIGEN_IF_CONSTEXPR((std::is_same >::value)) Dtype = SLU_C; - else EIGEN_IF_CONSTEXPR((std::is_same >::value)) Dtype = SLU_Z; + EIGEN_IF_CONSTEXPR ((std::is_same::value)) + Dtype = SLU_S; + else EIGEN_IF_CONSTEXPR ((std::is_same::value)) + Dtype = SLU_D; + else EIGEN_IF_CONSTEXPR ((std::is_same >::value)) + Dtype = SLU_C; + else EIGEN_IF_CONSTEXPR ((std::is_same >::value)) + Dtype = SLU_Z; else { eigen_assert(false && "Scalar type not supported by SuperLU"); } @@ -187,12 +190,11 @@ struct SluMatrix : SuperMatrix { static SluMatrix Map(SparseMatrixBase &a_mat) { MatrixType &mat(a_mat.derived()); SluMatrix res; - EIGEN_IF_CONSTEXPR((MatrixType::Flags & RowMajorBit) == RowMajorBit) { + EIGEN_IF_CONSTEXPR ((MatrixType::Flags & RowMajorBit) == RowMajorBit) { res.setStorageType(SLU_NR); res.nrow = internal::convert_index(mat.cols()); res.ncol = internal::convert_index(mat.rows()); - } - else { + } else { res.setStorageType(SLU_NC); res.nrow = internal::convert_index(mat.rows()); res.ncol = internal::convert_index(mat.cols()); @@ -208,8 +210,8 @@ struct SluMatrix : SuperMatrix { res.setScalarType(); // FIXME: the following type mapping is approximate. - EIGEN_IF_CONSTEXPR(int(MatrixType::Flags) & int(Upper)) res.Mtype = SLU_TRU; - EIGEN_IF_CONSTEXPR(int(MatrixType::Flags) & int(Lower)) res.Mtype = SLU_TRL; + EIGEN_IF_CONSTEXPR (int(MatrixType::Flags) & int(Upper)) res.Mtype = SLU_TRU; + EIGEN_IF_CONSTEXPR (int(MatrixType::Flags) & int(Lower)) res.Mtype = SLU_TRL; eigen_assert(((int(MatrixType::Flags) & int(SelfAdjoint)) == 0) && "SelfAdjoint matrix shape not supported by SuperLU"); @@ -239,12 +241,11 @@ template struct SluMatrixMapHelper > { typedef Derived MatrixType; static void run(MatrixType &mat, SluMatrix &res) { - EIGEN_IF_CONSTEXPR((MatrixType::Flags & RowMajorBit) == RowMajorBit) { + EIGEN_IF_CONSTEXPR ((MatrixType::Flags & RowMajorBit) == RowMajorBit) { res.setStorageType(SLU_NR); res.nrow = mat.cols(); res.ncol = mat.rows(); - } - else { + } else { res.setStorageType(SLU_NC); res.nrow = mat.rows(); res.ncol = mat.cols(); @@ -260,8 +261,8 @@ struct SluMatrixMapHelper > { res.setScalarType(); // FIXME: the following type mapping is approximate. - EIGEN_IF_CONSTEXPR(MatrixType::Flags & Upper) res.Mtype = SLU_TRU; - EIGEN_IF_CONSTEXPR(MatrixType::Flags & Lower) res.Mtype = SLU_TRL; + EIGEN_IF_CONSTEXPR (MatrixType::Flags & Upper) res.Mtype = SLU_TRU; + EIGEN_IF_CONSTEXPR (MatrixType::Flags & Lower) res.Mtype = SLU_TRL; eigen_assert(((MatrixType::Flags & SelfAdjoint) == 0) && "SelfAdjoint matrix shape not supported by SuperLU"); } diff --git a/Eigen/src/ThreadPool/RunQueue.h b/Eigen/src/ThreadPool/RunQueue.h index 7b73f5203..5b346d414 100644 --- a/Eigen/src/ThreadPool/RunQueue.h +++ b/Eigen/src/ThreadPool/RunQueue.h @@ -197,8 +197,9 @@ class RunQueue { std::atomic_thread_fence(std::memory_order_acquire); continue; } - EIGEN_IF_CONSTEXPR(NeedSizeEstimate) { return CalculateSize(front, back); } - else { + EIGEN_IF_CONSTEXPR (NeedSizeEstimate) { + return CalculateSize(front, back); + } else { // This value will be 0 if the queue is empty, and undefined otherwise. unsigned maybe_zero = ((front ^ back) & kMask2); // Queue size estimate must agree with maybe zero check on the queue diff --git a/blas/BandTriangularSolver.h b/blas/BandTriangularSolver.h index dcca68cad..7af568433 100644 --- a/blas/BandTriangularSolver.h +++ b/blas/BandTriangularSolver.h @@ -45,7 +45,7 @@ struct band_solve_triangular_selector 0) other.col(col).segment(IsLower ? i + 1 : i - actual_k, actual_k) -= diff --git a/blas/PackedTriangularMatrixVector.h b/blas/PackedTriangularMatrixVector.h index 0e7ac545b..b3ee50a0d 100644 --- a/blas/PackedTriangularMatrixVector.h +++ b/blas/PackedTriangularMatrixVector.h @@ -38,7 +38,9 @@ struct packed_triangular_matrix_vector_product 0)) { ResMap(res + (IsLower ? s + i : 0), r) += alpha * cj(rhs[i]) * ConjLhsType(LhsMap(lhs + s, r)); } - EIGEN_IF_CONSTEXPR(HasUnitDiag) { res[i] += alpha * cj(rhs[i]); } + EIGEN_IF_CONSTEXPR (HasUnitDiag) { + res[i] += alpha * cj(rhs[i]); + } lhs += IsLower ? size - i : i + 1; } }; @@ -67,7 +69,9 @@ struct packed_triangular_matrix_vector_product >(rhs + (IsLower ? 0 : i + 1), pi))) .sum(); - EIGEN_IF_CONSTEXPR(!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? i : 0]); + EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) rhs[i] /= cj(lhs[IsLower ? i : 0]); IsLower ? lhs += pi + 1 : lhs -= pi + 2; } } @@ -53,7 +53,7 @@ struct packed_triangular_solve_vector 0) Map >(rhs + (IsLower ? i + 1 : 0), r) -= rhs[i] * ConjLhsType(LhsMap(lhs + (IsLower ? 1 : 0), r)); diff --git a/unsupported/Eigen/src/AutoDiff/CoherentPadOp.h b/unsupported/Eigen/src/AutoDiff/CoherentPadOp.h index f8d73128a..59733c41a 100644 --- a/unsupported/Eigen/src/AutoDiff/CoherentPadOp.h +++ b/unsupported/Eigen/src/AutoDiff/CoherentPadOp.h @@ -92,12 +92,11 @@ struct unary_evaluator> : m_arg(pad.nestedExpression()), m_argImpl(m_arg), m_size(pad.nestedExpression().size()) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { - EIGEN_IF_CONSTEXPR(XprType::IsRowMajor) { + EIGEN_IF_CONSTEXPR (XprType::IsRowMajor) { if (col < m_size.value()) { return m_argImpl.coeff(1, col); } - } - else { + } else { if (row < m_size.value()) { return m_argImpl.coeff(row, 1); } @@ -116,7 +115,9 @@ struct unary_evaluator> EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { // AutoDiff scalar's derivative must be a vector, which is enforced by static assert. // Defer to linear access for simplicity. - EIGEN_IF_CONSTEXPR(XprType::IsRowMajor) { return packet(col); } + EIGEN_IF_CONSTEXPR (XprType::IsRowMajor) { + return packet(col); + } return packet(row); } diff --git a/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h b/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h index 914d682a5..b45241b28 100644 --- a/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h +++ b/unsupported/Eigen/src/SparseExtra/BlockSparseMatrix.h @@ -94,8 +94,9 @@ template struct TripletComp { typedef typename Iterator::value_type Triplet; bool operator()(const Triplet& a, const Triplet& b) { - EIGEN_IF_CONSTEXPR(IsColMajor) { return ((a.col() == b.col() && a.row() < b.row()) || (a.col() < b.col())); } - else { + EIGEN_IF_CONSTEXPR (IsColMajor) { + return ((a.col() == b.col() && a.row() < b.row()) || (a.col() < b.col())); + } else { return ((a.row() == b.row() && a.col() < b.col()) || (a.row() < b.row())); } } diff --git a/unsupported/Eigen/src/SparseExtra/MarketIO.h b/unsupported/Eigen/src/SparseExtra/MarketIO.h index a8dabbe8b..84a8020f4 100644 --- a/unsupported/Eigen/src/SparseExtra/MarketIO.h +++ b/unsupported/Eigen/src/SparseExtra/MarketIO.h @@ -72,8 +72,8 @@ inline void GetDenseElt(const std::string& line, std::complex& val) template inline void putMarketHeader(std::string& header, int sym) { header = "%%MatrixMarket matrix coordinate "; - EIGEN_IF_CONSTEXPR( - (std::is_same >::value || std::is_same >::value)) { + EIGEN_IF_CONSTEXPR ((std::is_same >::value || + std::is_same >::value)) { header += " complex"; if (sym == Symmetric) header += " symmetric"; @@ -81,8 +81,7 @@ inline void putMarketHeader(std::string& header, int sym) { header += " Hermitian"; else header += " general"; - } - else { + } else { header += " real"; if (sym == Symmetric) header += " symmetric"; @@ -340,11 +339,10 @@ bool saveMarketDense(const DenseType& mat, const std::string& filename) { out.flags(std::ios_base::scientific); out.precision(std::numeric_limits::digits10 + 2); - EIGEN_IF_CONSTEXPR( - (std::is_same >::value || std::is_same >::value)) { + EIGEN_IF_CONSTEXPR ((std::is_same >::value || + std::is_same >::value)) { out << "%%MatrixMarket matrix array complex general\n"; - } - else { + } else { out << "%%MatrixMarket matrix array real general\n"; } out << mat.rows() << " " << mat.cols() << "\n"; diff --git a/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h b/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h index e1d9d542f..593511b1d 100644 --- a/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h +++ b/unsupported/Eigen/src/SparseExtra/MatrixMarketIterator.h @@ -179,12 +179,14 @@ class MatrixMarketIterator { if (!getMarketHeader(curfile, m_sym, iscomplex, isvector)) continue; if (isvector) continue; if (!iscomplex) { - EIGEN_IF_CONSTEXPR( - (std::is_same >::value || std::is_same >::value)) - continue; + EIGEN_IF_CONSTEXPR ((std::is_same >::value || + std::is_same >::value)) + continue; } if (iscomplex) { - EIGEN_IF_CONSTEXPR((std::is_same::value || std::is_same::value)) { continue; } + EIGEN_IF_CONSTEXPR ((std::is_same::value || std::is_same::value)) { + continue; + } } // Get the matrix name diff --git a/unsupported/Eigen/src/Tensor/Tensor.h b/unsupported/Eigen/src/Tensor/Tensor.h index 134e1b9ea..3a0c732d7 100644 --- a/unsupported/Eigen/src/Tensor/Tensor.h +++ b/unsupported/Eigen/src/Tensor/Tensor.h @@ -363,8 +363,9 @@ class Tensor : public TensorBase& indices) const { - EIGEN_IF_CONSTEXPR(Options & RowMajor) { return m_storage.dimensions().IndexOfRowMajor(indices); } - else { + EIGEN_IF_CONSTEXPR (Options & RowMajor) { + return m_storage.dimensions().IndexOfRowMajor(indices); + } else { return m_storage.dimensions().IndexOfColMajor(indices); } } diff --git a/unsupported/Eigen/src/Tensor/TensorArgMax.h b/unsupported/Eigen/src/Tensor/TensorArgMax.h index 8bfe8ac04..f9b5ad256 100644 --- a/unsupported/Eigen/src/Tensor/TensorArgMax.h +++ b/unsupported/Eigen/src/Tensor/TensorArgMax.h @@ -214,11 +214,10 @@ struct TensorEvaluator, Devic m_impl(op.expression().index_pairs().reduce(op.reduce_dims(), op.reduce_op()), device), m_return_dim(op.return_dim()) { gen_strides(m_orig_impl.dimensions(), m_strides); - EIGEN_IF_CONSTEXPR(Layout == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (Layout == static_cast(ColMajor)) { const Index total_size = internal::array_prod(m_orig_impl.dimensions()); m_stride_mod = (m_return_dim < NumDims - 1) ? m_strides[m_return_dim + 1] : total_size; - } - else { + } else { const Index total_size = internal::array_prod(m_orig_impl.dimensions()); m_stride_mod = (m_return_dim > 0) ? m_strides[m_return_dim - 1] : total_size; } @@ -257,13 +256,12 @@ struct TensorEvaluator, Devic // Calculate m_stride_div and m_stride_mod, which are used to // calculate the value of an index w.r.t. the m_return_dim. - EIGEN_IF_CONSTEXPR(Layout == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (Layout == static_cast(ColMajor)) { strides[0] = 1; for (int i = 1; i < NumDims; ++i) { strides[i] = strides[i - 1] * dims[i - 1]; } - } - else { + } else { strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { strides[i] = strides[i + 1] * dims[i + 1]; diff --git a/unsupported/Eigen/src/Tensor/TensorBlock.h b/unsupported/Eigen/src/Tensor/TensorBlock.h index 30b628377..3d43a0a7b 100644 --- a/unsupported/Eigen/src/Tensor/TensorBlock.h +++ b/unsupported/Eigen/src/Tensor/TensorBlock.h @@ -34,13 +34,12 @@ EIGEN_ALWAYS_INLINE DSizes strides(const DSizes(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { strides[0] = 1; for (int i = 1; i < NumDims; ++i) { strides[i] = strides[i - 1] * dimensions[i - 1]; } - } - else { + } else { strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { strides[i] = strides[i + 1] * dimensions[i + 1]; @@ -952,7 +951,7 @@ class StridedLinearBufferCopy { const Scalar* src = &src_data[src_offset]; Scalar* dst = &dst_data[dst_offset]; - EIGEN_IF_CONSTEXPR(!Vectorizable) { + EIGEN_IF_CONSTEXPR (!Vectorizable) { for (Index i = 0; i < count; ++i) { dst[i * dst_stride] = src[i * src_stride]; } @@ -962,7 +961,7 @@ class StridedLinearBufferCopy { const IndexType vectorized_size = PacketSize * (count / PacketSize); IndexType i = 0; - EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Linear) { + EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Linear) { // ******************************************************************** // // Linear copy from `src` to `dst`. const IndexType unrolled_size = (4 * PacketSize) * (count / (4 * PacketSize)); @@ -977,7 +976,7 @@ class StridedLinearBufferCopy { Packet p = ploadu(src + i); pstoreu(dst + i, p); } - EIGEN_IF_CONSTEXPR(HasHalfPacket) { + EIGEN_IF_CONSTEXPR (HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket p = ploadu(src + i); @@ -989,15 +988,14 @@ class StridedLinearBufferCopy { dst[i] = src[i]; } // ******************************************************************** // - } - else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Scatter) { + } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Scatter) { // Scatter from `src` to `dst`. eigen_assert(src_stride == 1 && dst_stride != 1); for (; i < vectorized_size; i += PacketSize) { Packet p = ploadu(src + i); pscatter(dst + i * dst_stride, p, dst_stride); } - EIGEN_IF_CONSTEXPR(HasHalfPacket) { + EIGEN_IF_CONSTEXPR (HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket p = ploadu(src + i); @@ -1009,8 +1007,7 @@ class StridedLinearBufferCopy { dst[i * dst_stride] = src[i]; } // ******************************************************************** // - } - else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::FillLinear) { + } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::FillLinear) { // Fill `dst` with value at `*src`. eigen_assert(src_stride == 0 && dst_stride == 1); @@ -1025,7 +1022,7 @@ class StridedLinearBufferCopy { for (; i < vectorized_size; i += PacketSize) { pstoreu(dst + i, p); } - EIGEN_IF_CONSTEXPR(HasHalfPacket) { + EIGEN_IF_CONSTEXPR (HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket hp = pset1(s); @@ -1037,8 +1034,7 @@ class StridedLinearBufferCopy { dst[i] = s; } // ******************************************************************** // - } - else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::FillScatter) { + } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::FillScatter) { // Scatter `*src` into `dst`. eigen_assert(src_stride == 0 && dst_stride != 1); Scalar s = *src; @@ -1046,7 +1042,7 @@ class StridedLinearBufferCopy { for (; i < vectorized_size; i += PacketSize) { pscatter(dst + i * dst_stride, p, dst_stride); } - EIGEN_IF_CONSTEXPR(HasHalfPacket) { + EIGEN_IF_CONSTEXPR (HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket hp = pset1(s); @@ -1058,15 +1054,14 @@ class StridedLinearBufferCopy { dst[i * dst_stride] = s; } // ******************************************************************** // - } - else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Gather) { + } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Gather) { // Gather from `src` into `dst`. eigen_assert(dst_stride == 1); for (; i < vectorized_size; i += PacketSize) { Packet p = pgather(src + i * src_stride, src_stride); pstoreu(dst + i, p); } - EIGEN_IF_CONSTEXPR(HasHalfPacket) { + EIGEN_IF_CONSTEXPR (HasHalfPacket) { const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize); if (i < vectorized_half_size) { HalfPacket p = pgather(src + i * src_stride, src_stride); @@ -1078,14 +1073,12 @@ class StridedLinearBufferCopy { dst[i] = src[i * src_stride]; } // ******************************************************************** // - } - else EIGEN_IF_CONSTEXPR(kind == StridedLinearBufferCopy::Kind::Random) { + } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Random) { // Random. for (; i < count; ++i) { dst[i * dst_stride] = src[i * src_stride]; } - } - else { + } else { eigen_assert(false); } } diff --git a/unsupported/Eigen/src/Tensor/TensorBroadcasting.h b/unsupported/Eigen/src/Tensor/TensorBroadcasting.h index a6fce2998..4589307ca 100644 --- a/unsupported/Eigen/src/Tensor/TensorBroadcasting.h +++ b/unsupported/Eigen/src/Tensor/TensorBroadcasting.h @@ -145,15 +145,14 @@ struct TensorEvaluator, Device> { } } - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_inputStrides[0] = 1; m_outputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1]; m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; } - } - else { + } else { m_inputStrides[NumDims - 1] = 1; m_outputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -218,14 +217,13 @@ struct TensorEvaluator, Device> { return m_impl.coeff(0); } - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { if (isCopy) { return m_impl.coeff(index); } else { return coeffColMajor(index); } - } - else { + } else { if (isCopy) { return m_impl.coeff(index); } else { @@ -314,7 +312,7 @@ struct TensorEvaluator, Device> { return internal::pset1(m_impl.coeff(0)); } - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { if (isCopy) { #ifdef EIGEN_GPU_COMPILE_PHASE // See PR 437: on NVIDIA P100 and K20m we observed a x3-4 speed up by enforcing @@ -332,8 +330,7 @@ struct TensorEvaluator, Device> { } else { return packetColMajor(index); } - } - else { + } else { if (isCopy) { #ifdef EIGEN_GPU_COMPILE_PHASE // See above. @@ -361,11 +358,10 @@ struct TensorEvaluator, Device> { Index startDim, endDim; Index inputIndex, outputOffset, batchedIndex; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { startDim = NumDims - 1; endDim = 1; - } - else { + } else { startDim = 0; endDim = NumDims - 2; } diff --git a/unsupported/Eigen/src/Tensor/TensorChipping.h b/unsupported/Eigen/src/Tensor/TensorChipping.h index a95de2fd2..876376ee9 100644 --- a/unsupported/Eigen/src/Tensor/TensorChipping.h +++ b/unsupported/Eigen/src/Tensor/TensorChipping.h @@ -156,13 +156,12 @@ struct TensorEvaluator, Device> { m_stride = 1; m_inputStride = 1; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < m_dim.actualDim(); ++i) { m_stride *= input_dims[i]; m_inputStride *= input_dims[i]; } - } - else { + } else { for (int i = NumInputDims - 1; i > m_dim.actualDim(); --i) { m_stride *= input_dims[i]; m_inputStride *= input_dims[i]; @@ -183,11 +182,10 @@ struct TensorEvaluator, Device> { before_chipped_dim_product *= input_dims[i]; } - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_isEffectivelyInnerChipping = before_chipped_dim_product == 1; m_isEffectivelyOuterChipping = after_chipped_dim_product == 1; - } - else { + } else { m_isEffectivelyInnerChipping = after_chipped_dim_product == 1; m_isEffectivelyOuterChipping = before_chipped_dim_product == 1; } diff --git a/unsupported/Eigen/src/Tensor/TensorConcatenation.h b/unsupported/Eigen/src/Tensor/TensorConcatenation.h index cc5146b8a..e0aef67d6 100644 --- a/unsupported/Eigen/src/Tensor/TensorConcatenation.h +++ b/unsupported/Eigen/src/Tensor/TensorConcatenation.h @@ -164,7 +164,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_leftStrides[0] = 1; m_rightStrides[0] = 1; m_outputStrides[0] = 1; @@ -174,8 +174,7 @@ struct TensorEvaluator subs; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { subs[i] = index / m_outputStrides[i]; index -= subs[i] * m_outputStrides[i]; } subs[0] = index; - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { subs[i] = index / m_outputStrides[i]; index -= subs[i] * m_outputStrides[i]; @@ -379,14 +377,13 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { left_index = subs[0]; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { left_index += (subs[i] % left_dims[i]) * m_leftStrides[i]; } - } - else { + } else { left_index = subs[NumDims - 1]; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { @@ -398,14 +395,13 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { right_index = subs[0]; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { right_index += (subs[i] % right_dims[i]) * m_rightStrides[i]; } - } - else { + } else { right_index = subs[NumDims - 1]; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { @@ -436,7 +432,7 @@ struct TensorEvaluator subs_end; Index remaining = index; Index remaining_end = index + packetSize - 1; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { subs[i] = remaining / m_outputStrides[i]; remaining -= subs[i] * m_outputStrides[i]; @@ -445,8 +441,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { left_index = subs[0]; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { left_index += subs[i] * m_leftStrides[i]; } - } - else { + } else { left_index = subs[NumDims - 1]; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { @@ -495,14 +489,13 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { right_index = subs[0]; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { right_index += subs[i] * m_rightStrides[i]; } - } - else { + } else { right_index = subs[NumDims - 1]; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { diff --git a/unsupported/Eigen/src/Tensor/TensorContraction.h b/unsupported/Eigen/src/Tensor/TensorContraction.h index 22f924876..99c51fe5d 100644 --- a/unsupported/Eigen/src/Tensor/TensorContraction.h +++ b/unsupported/Eigen/src/Tensor/TensorContraction.h @@ -466,7 +466,7 @@ struct TensorContractionEvaluatorBase { DSizes eval_left_dims; DSizes eval_right_dims; array, ContractDims> eval_op_indices; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { // For ColMajor, we keep using the existing dimensions for (int i = 0; i < LDims; i++) { eval_left_dims[i] = m_leftImpl.dimensions()[i]; @@ -479,8 +479,7 @@ struct TensorContractionEvaluatorBase { eval_op_indices[i].first = op.indices()[i].first; eval_op_indices[i].second = op.indices()[i].second; } - } - else { + } else { // For RowMajor, we need to reverse the existing dimensions for (int i = 0; i < LDims; i++) { eval_left_dims[i] = m_leftImpl.dimensions()[LDims - i - 1]; @@ -642,7 +641,7 @@ struct TensorContractionEvaluatorBase { } // If the layout is RowMajor, we need to reverse the m_dimensions - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(RowMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(RowMajor)) { for (int i = 0, j = NumDims - 1; i < j; i++, j--) { numext::swap(m_dimensions[i], m_dimensions[j]); } diff --git a/unsupported/Eigen/src/Tensor/TensorContractionThreadPool.h b/unsupported/Eigen/src/Tensor/TensorContractionThreadPool.h index e2473d7aa..b8975035b 100644 --- a/unsupported/Eigen/src/Tensor/TensorContractionThreadPool.h +++ b/unsupported/Eigen/src/Tensor/TensorContractionThreadPool.h @@ -265,13 +265,12 @@ struct TensorEvaluator ctx( this, num_threads, buffer, m, n, k, bm, bn, bk, nm, nn, nk, gm, gn, nm0, nn0, shard_by_col, parallel_pack, parallelize_by_sharding_dim_only, NoCallback()); ctx.run(); - } - else { + } else { auto* ctx = new EvalParallelContext( this, num_threads, buffer, m, n, k, bm, bn, bk, nm, nn, nk, gm, gn, nm0, nn0, shard_by_col, parallel_pack, parallelize_by_sharding_dim_only, std::move(done)); diff --git a/unsupported/Eigen/src/Tensor/TensorConvolution.h b/unsupported/Eigen/src/Tensor/TensorConvolution.h index 78fc0820f..513c87b4d 100644 --- a/unsupported/Eigen/src/Tensor/TensorConvolution.h +++ b/unsupported/Eigen/src/Tensor/TensorConvolution.h @@ -34,15 +34,14 @@ class IndexMapper { array inputStrides; array outputStrides; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { inputStrides[0] = 1; outputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { inputStrides[i] = inputStrides[i - 1] * input_dims[i - 1]; outputStrides[i] = outputStrides[i - 1] * dimensions[i - 1]; } - } - else { + } else { inputStrides[NumDims - 1] = 1; outputStrides[NumDims - 1] = 1; for (int i = static_cast(NumDims) - 2; i >= 0; --i) { @@ -79,7 +78,7 @@ class IndexMapper { m_outputStrides[i] = outputStrides[ordering[i]]; } - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumDims; ++i) { if (i > NumKernelDims) { m_gpuInputStrides[i] = m_gpuInputStrides[i - 1] * gpuInputDimensions[i - 1]; @@ -89,8 +88,7 @@ class IndexMapper { m_gpuOutputStrides[i] = 1; } } - } - else { + } else { for (int i = NumDims - 1; i >= 0; --i) { if (i + 1 < static_cast(offset)) { m_gpuInputStrides[i] = m_gpuInputStrides[i + 1] * gpuInputDimensions[i + 1]; @@ -105,7 +103,7 @@ class IndexMapper { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputPlaneToTensorInputOffset(Index p) const { Index inputIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int d = NumDims - 1; d > NumKernelDims; --d) { const Index idx = p / m_gpuInputStrides[d]; inputIndex += idx * m_inputStrides[d]; @@ -114,8 +112,7 @@ class IndexMapper { if (NumKernelDims < NumDims) { inputIndex += p * m_inputStrides[NumKernelDims]; } - } - else { + } else { std::ptrdiff_t limit = 0; if (NumKernelDims < NumDims) { limit = NumDims - NumKernelDims - 1; @@ -132,7 +129,7 @@ class IndexMapper { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputPlaneToTensorOutputOffset(Index p) const { Index outputIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int d = NumDims - 1; d > NumKernelDims; --d) { const Index idx = p / m_gpuOutputStrides[d]; outputIndex += idx * m_outputStrides[d]; @@ -141,8 +138,7 @@ class IndexMapper { if (NumKernelDims < NumDims) { outputIndex += p * m_outputStrides[NumKernelDims]; } - } - else { + } else { std::ptrdiff_t limit = 0; if (NumKernelDims < NumDims) { limit = NumDims - NumKernelDims - 1; @@ -314,13 +310,12 @@ struct TensorEvaluator::Dimensions& input_dims = m_inputImpl.dimensions(); const typename TensorEvaluator::Dimensions& kernel_dims = m_kernelImpl.dimensions(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_inputStride[0] = 1; for (int i = 1; i < NumDims; ++i) { m_inputStride[i] = m_inputStride[i - 1] * input_dims[i - 1]; } - } - else { + } else { m_inputStride[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_inputStride[i] = m_inputStride[i + 1] * input_dims[i + 1]; @@ -328,7 +323,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumKernelDims; ++i) { const Index index = op.indices()[i]; const Index input_dim = input_dims[index]; @@ -347,8 +342,7 @@ struct TensorEvaluator= 0; --i) { const Index index = op.indices()[i]; const Index input_dim = input_dims[index]; @@ -404,7 +398,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / m_outputStride[i]; const Index idx1 = indices[1] / m_outputStride[i]; @@ -413,8 +407,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStride[i]; startInput += idx * m_inputStride[i]; index -= idx * m_outputStride[i]; } - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStride[i]; startInput += idx * m_inputStride[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorEvaluator.h b/unsupported/Eigen/src/Tensor/TensorEvaluator.h index d5fe3396a..04b01e99c 100644 --- a/unsupported/Eigen/src/Tensor/TensorEvaluator.h +++ b/unsupported/Eigen/src/Tensor/TensorEvaluator.h @@ -120,20 +120,18 @@ struct TensorEvaluator { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array& coords) const { eigen_assert(m_data != NULL); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { return m_data[m_dims.IndexOfColMajor(coords)]; - } - else { + } else { return m_data[m_dims.IndexOfRowMajor(coords)]; } } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(const array& coords) const { eigen_assert(m_data != NULL); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { return m_data[m_dims.IndexOfColMajor(coords)]; - } - else { + } else { return m_data[m_dims.IndexOfRowMajor(coords)]; } } diff --git a/unsupported/Eigen/src/Tensor/TensorFFT.h b/unsupported/Eigen/src/Tensor/TensorFFT.h index a37fbd7c2..3c06e47f8 100644 --- a/unsupported/Eigen/src/Tensor/TensorFFT.h +++ b/unsupported/Eigen/src/Tensor/TensorFFT.h @@ -179,13 +179,12 @@ struct TensorEvaluator, D m_dimensions[i] = input_dims[i]; } - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_strides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1]; } - } - else { + } else { m_strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1]; @@ -644,7 +643,7 @@ struct TensorEvaluator, D EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index getBaseOffsetFromIndex(Index index, Index omitted_dim) const { Index result = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > omitted_dim; --i) { const Index partial_m_stride = m_strides[i] / m_dimensions[omitted_dim]; const Index idx = index / partial_m_stride; @@ -652,8 +651,7 @@ struct TensorEvaluator, D result += idx * m_strides[i]; } result += index; - } - else { + } else { for (Index i = 0; i < omitted_dim; ++i) { const Index partial_m_stride = m_strides[i] / m_dimensions[omitted_dim]; const Index idx = index / partial_m_stride; diff --git a/unsupported/Eigen/src/Tensor/TensorFixedSize.h b/unsupported/Eigen/src/Tensor/TensorFixedSize.h index 696baf297..ba42f3b89 100644 --- a/unsupported/Eigen/src/Tensor/TensorFixedSize.h +++ b/unsupported/Eigen/src/Tensor/TensorFixedSize.h @@ -203,8 +203,9 @@ class TensorFixedSize : public TensorBase& indices) const { - EIGEN_IF_CONSTEXPR(Options & RowMajor) { return m_storage.dimensions().IndexOfRowMajor(indices); } - else { + EIGEN_IF_CONSTEXPR (Options & RowMajor) { + return m_storage.dimensions().IndexOfRowMajor(indices); + } else { return m_storage.dimensions().IndexOfColMajor(indices); } } diff --git a/unsupported/Eigen/src/Tensor/TensorGenerator.h b/unsupported/Eigen/src/Tensor/TensorGenerator.h index 6f7b7b9c5..0b46bcb3d 100644 --- a/unsupported/Eigen/src/Tensor/TensorGenerator.h +++ b/unsupported/Eigen/src/Tensor/TensorGenerator.h @@ -105,15 +105,14 @@ struct TensorEvaluator, Device> { TensorEvaluator argImpl(op.expression(), device); m_dimensions = argImpl.dimensions(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_strides[0] = 1; EIGEN_UNROLL_LOOP for (int i = 1; i < NumDims; ++i) { m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1]; if (m_strides[i] != 0) m_fast_strides[i] = IndexDivisor(m_strides[i]); } - } - else { + } else { m_strides[NumDims - 1] = 1; EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i >= 0; --i) { @@ -242,15 +241,14 @@ struct TensorEvaluator, Device> { protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void extract_coordinates(Index index, array& coords) const { - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_fast_strides[i]; index -= idx * m_strides[i]; coords[i] = idx; } coords[0] = index; - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_fast_strides[i]; index -= idx * m_strides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorIO.h b/unsupported/Eigen/src/Tensor/TensorIO.h index 513864bc3..638f244d1 100644 --- a/unsupported/Eigen/src/Tensor/TensorIO.h +++ b/unsupported/Eigen/src/Tensor/TensorIO.h @@ -238,8 +238,9 @@ struct TensorPrinter { if (fmt.precision == StreamPrecision) { explicit_precision = 0; } else if (fmt.precision == FullPrecision) { - EIGEN_IF_CONSTEXPR(NumTraits::IsInteger) { explicit_precision = 0; } - else { + EIGEN_IF_CONSTEXPR (NumTraits::IsInteger) { + explicit_precision = 0; + } else { explicit_precision = significant_decimals_impl::run(); } } else { @@ -385,8 +386,9 @@ struct TensorPrinter { if (fmt.precision == StreamPrecision) { explicit_precision = 0; } else if (fmt.precision == FullPrecision) { - EIGEN_IF_CONSTEXPR(NumTraits::IsInteger) { explicit_precision = 0; } - else { + EIGEN_IF_CONSTEXPR (NumTraits::IsInteger) { + explicit_precision = 0; + } else { explicit_precision = significant_decimals_impl::run(); } } else { diff --git a/unsupported/Eigen/src/Tensor/TensorImagePatch.h b/unsupported/Eigen/src/Tensor/TensorImagePatch.h index ddcc9335d..18aac8d22 100644 --- a/unsupported/Eigen/src/Tensor/TensorImagePatch.h +++ b/unsupported/Eigen/src/Tensor/TensorImagePatch.h @@ -193,12 +193,11 @@ struct TensorEvaluator, Device> { const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); // Caches a few variables. - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_inputDepth = input_dims[0]; m_inputRows = input_dims[1]; m_inputCols = input_dims[2]; - } - else { + } else { m_inputDepth = input_dims[NumInputDims - 1]; m_inputRows = input_dims[NumInputDims - 2]; m_inputCols = input_dims[NumInputDims - 3]; @@ -270,7 +269,7 @@ struct TensorEvaluator, Device> { eigen_assert(m_outputCols > 0); // Dimensions for result of extraction. - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { // ColMajor // 0: depth // 1: patch_rows @@ -284,8 +283,7 @@ struct TensorEvaluator, Device> { for (int i = 4; i < NumDims; ++i) { m_dimensions[i] = input_dims[i - 1]; } - } - else { + } else { // RowMajor // NumDims-1: depth // NumDims-2: patch_rows @@ -302,12 +300,11 @@ struct TensorEvaluator, Device> { } // Strides for moving the patch in various dimensions. - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_colStride = m_dimensions[1]; m_patchStride = m_colStride * m_dimensions[2] * m_dimensions[0]; m_otherStride = m_patchStride * m_dimensions[3]; - } - else { + } else { m_colStride = m_dimensions[NumDims - 2]; m_patchStride = m_colStride * m_dimensions[NumDims - 3] * m_dimensions[NumDims - 1]; m_otherStride = m_patchStride * m_dimensions[NumDims - 4]; @@ -328,10 +325,9 @@ struct TensorEvaluator, Device> { // Number of patches in the width dimension. m_fastOutputRows = internal::TensorIntDivisor(m_outputRows); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_fastOutputDepth = internal::TensorIntDivisor(m_dimensions[0]); - } - else { + } else { m_fastOutputDepth = internal::TensorIntDivisor(m_dimensions[NumDims - 1]); } } diff --git a/unsupported/Eigen/src/Tensor/TensorInflation.h b/unsupported/Eigen/src/Tensor/TensorInflation.h index 2550d12ac..e759edd7d 100644 --- a/unsupported/Eigen/src/Tensor/TensorInflation.h +++ b/unsupported/Eigen/src/Tensor/TensorInflation.h @@ -111,15 +111,14 @@ struct TensorEvaluator, Device> { } const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_outputStrides[0] = 1; m_inputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1]; } - } - else { // RowMajor + } else { // RowMajor m_outputStrides[NumDims - 1] = 1; m_inputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -142,7 +141,7 @@ struct TensorEvaluator, Device> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool getInputIndex(Index index, Index* inputIndex) const { eigen_assert(index < dimensions().TotalSize()); *inputIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; @@ -157,8 +156,7 @@ struct TensorEvaluator, Device> { } *inputIndex += index / m_strides[0]; return true; - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorMap.h b/unsupported/Eigen/src/Tensor/TensorMap.h index eec3744dc..992506105 100644 --- a/unsupported/Eigen/src/Tensor/TensorMap.h +++ b/unsupported/Eigen/src/Tensor/TensorMap.h @@ -105,11 +105,10 @@ class TensorMap : public TensorBase& indices) const { - EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(indices); return m_data[index]; - } - else { + } else { const Index index = m_dimensions.IndexOfColMajor(indices); return m_data[index]; } @@ -130,12 +129,11 @@ class TensorMap : public TensorBase::highest() >= otherIndices)...)); - EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(array{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; - } - else { + } else { const Index index = m_dimensions.IndexOfColMajor(array{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; @@ -143,11 +141,10 @@ class TensorMap : public TensorBase& indices) { - EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(indices); return m_data[index]; - } - else { + } else { const Index index = m_dimensions.IndexOfColMajor(indices); return m_data[index]; } @@ -170,12 +167,11 @@ class TensorMap : public TensorBase::highest() >= otherIndices)...)); const std::size_t NumDims = sizeof...(otherIndices) + 2; - EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) { const Index index = m_dimensions.IndexOfRowMajor(array{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; - } - else { + } else { const Index index = m_dimensions.IndexOfColMajor(array{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; diff --git a/unsupported/Eigen/src/Tensor/TensorMorphing.h b/unsupported/Eigen/src/Tensor/TensorMorphing.h index 2be1637ae..8d9ac9d5f 100644 --- a/unsupported/Eigen/src/Tensor/TensorMorphing.h +++ b/unsupported/Eigen/src/Tensor/TensorMorphing.h @@ -399,7 +399,7 @@ struct TensorEvaluator, Devi const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); const Sizes& output_dims = op.sizes(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_inputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_inputStrides[i] = m_inputStrides[i - 1] * input_dims[i - 1]; @@ -411,8 +411,7 @@ struct TensorEvaluator, Devi m_outputStrides[i] = m_outputStrides[i - 1] * output_dims[i - 1]; m_fastOutputStrides[i] = internal::TensorIntDivisor(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1); } - } - else { + } else { m_inputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_inputStrides[i] = m_inputStrides[i + 1] * input_dims[i + 1]; @@ -433,15 +432,14 @@ struct TensorEvaluator, Devi m_impl.evalSubExprsIfNeeded(NULL); if (!NumTraits>::RequireInitialization && data && m_impl.data()) { Index contiguous_values = 1; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumDims; ++i) { contiguous_values *= dimensions()[i]; if (dimensions()[i] != m_impl.dimensions()[i]) { break; } } - } - else { + } else { for (int i = NumDims - 1; i >= 0; --i) { contiguous_values *= dimensions()[i]; if (dimensions()[i] != m_impl.dimensions()[i]) { @@ -493,7 +491,7 @@ struct TensorEvaluator, Devi Index inputIndices[] = {0, 0}; Index indices[] = {index, index + packetSize - 1}; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / m_fastOutputStrides[i]; @@ -505,8 +503,7 @@ struct TensorEvaluator, Devi } inputIndices[0] += (indices[0] + m_offsets[0]); inputIndices[1] += (indices[1] + m_offsets[0]); - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx0 = indices[0] / m_fastOutputStrides[i]; @@ -557,7 +554,7 @@ struct TensorEvaluator, Devi typename Storage::Type result = constCast(m_impl.data()); if (result) { Index offset = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumDims; ++i) { if (m_dimensions[i] != m_impl.dimensions()[i]) { offset += m_offsets[i] * m_inputStrides[i]; @@ -570,8 +567,7 @@ struct TensorEvaluator, Devi break; } } - } - else { + } else { for (int i = NumDims - 1; i >= 0; --i) { if (m_dimensions[i] != m_impl.dimensions()[i]) { offset += m_offsets[i] * m_inputStrides[i]; @@ -593,7 +589,7 @@ struct TensorEvaluator, Devi protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const { Index inputIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_fastOutputStrides[i]; @@ -601,8 +597,7 @@ struct TensorEvaluator, Devi index -= idx * m_outputStrides[i]; } inputIndex += (index + m_offsets[0]); - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_fastOutputStrides[i]; @@ -675,7 +670,7 @@ struct TensorEvaluator, Device> const int packetSize = PacketType::size; Index inputIndices[] = {0, 0}; Index indices[] = {index, index + packetSize - 1}; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / this->m_fastOutputStrides[i]; @@ -687,8 +682,7 @@ struct TensorEvaluator, Device> } inputIndices[0] += (indices[0] + this->m_offsets[0]); inputIndices[1] += (indices[1] + this->m_offsets[0]); - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx0 = indices[0] / this->m_fastOutputStrides[i]; @@ -844,7 +838,7 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_inputStrides[0] = m_strides[0]; m_offsets[0] = startIndicesClamped[0]; Index previousDimProduct = 1; @@ -860,8 +854,7 @@ struct TensorEvaluator(m_outputStrides[i] > 0 ? m_outputStrides[i] : 1); } - } - else { + } else { m_inputStrides[NumDims - 1] = m_strides[NumDims - 1]; m_offsets[NumDims - 1] = startIndicesClamped[NumDims - 1]; Index previousDimProduct = 1; @@ -905,15 +898,14 @@ struct TensorEvaluator(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i >= 0; --i) { const Index idx = index / m_fastOutputStrides[i]; inputIndex += idx * m_inputStrides[i] + m_offsets[i]; index -= idx * m_outputStrides[i]; } - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims; ++i) { const Index idx = index / m_fastOutputStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorPadding.h b/unsupported/Eigen/src/Tensor/TensorPadding.h index fda6932b9..a1079d561 100644 --- a/unsupported/Eigen/src/Tensor/TensorPadding.h +++ b/unsupported/Eigen/src/Tensor/TensorPadding.h @@ -121,7 +121,7 @@ struct TensorEvaluator, Device m_dimensions[i] += m_padding[i].first + m_padding[i].second; } const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_inputStrides[0] = 1; m_outputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { @@ -129,8 +129,7 @@ struct TensorEvaluator, Device m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; } m_outputStrides[NumDims] = m_outputStrides[NumDims - 1] * m_dimensions[NumDims - 1]; - } - else { + } else { m_inputStrides[NumDims - 1] = 1; m_outputStrides[NumDims] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -160,7 +159,7 @@ struct TensorEvaluator, Device EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { eigen_assert(index < dimensions().TotalSize()); Index inputIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; @@ -174,8 +173,7 @@ struct TensorEvaluator, Device return m_paddingValue; } inputIndex += (index - m_padding[0].first); - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStrides[i + 1]; @@ -195,17 +193,18 @@ struct TensorEvaluator, Device template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { return packetColMajor(index); } + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { + return packetColMajor(index); + } return packetRowMajor(index); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const { TensorOpCost cost = m_impl.costPerCoeff(vectorized); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims; ++i) updateCostPerDimension(cost, i, i == 0); - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i >= 0; --i) updateCostPerDimension(cost, i, i == NumDims - 1); } diff --git a/unsupported/Eigen/src/Tensor/TensorPatch.h b/unsupported/Eigen/src/Tensor/TensorPatch.h index 1826d9bd0..de84fd3bc 100644 --- a/unsupported/Eigen/src/Tensor/TensorPatch.h +++ b/unsupported/Eigen/src/Tensor/TensorPatch.h @@ -101,7 +101,7 @@ struct TensorEvaluator, Device> { Index num_patches = 1; const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); const PatchDim& patch_dims = op.patch_dims(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < NumDims - 1; ++i) { m_dimensions[i] = patch_dims[i]; num_patches *= (input_dims[i] - patch_dims[i] + 1); @@ -118,8 +118,7 @@ struct TensorEvaluator, Device> { for (int i = 1; i < NumDims; ++i) { m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; } - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { m_dimensions[i + 1] = patch_dims[i]; num_patches *= (input_dims[i] - patch_dims[i] + 1); @@ -155,7 +154,7 @@ struct TensorEvaluator, Device> { // Find the offset of the element wrt the location of the first element. Index patchOffset = index - patchIndex * m_outputStrides[output_stride_index]; Index inputIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i > 0; --i) { const Index patchIdx = patchIndex / m_patchStrides[i]; @@ -164,8 +163,7 @@ struct TensorEvaluator, Device> { patchOffset -= offsetIdx * m_outputStrides[i]; inputIndex += (patchIdx + offsetIdx) * m_inputStrides[i]; } - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 2; ++i) { const Index patchIdx = patchIndex / m_patchStrides[i]; @@ -191,7 +189,7 @@ struct TensorEvaluator, Device> { indices[1] - patchIndices[1] * m_outputStrides[output_stride_index]}; Index inputIndices[2] = {0, 0}; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 2; i > 0; --i) { const Index patchIdx[2] = {patchIndices[0] / m_patchStrides[i], patchIndices[1] / m_patchStrides[i]}; @@ -205,8 +203,7 @@ struct TensorEvaluator, Device> { inputIndices[0] += (patchIdx[0] + offsetIdx[0]) * m_inputStrides[i]; inputIndices[1] += (patchIdx[1] + offsetIdx[1]) * m_inputStrides[i]; } - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 2; ++i) { const Index patchIdx[2] = {patchIndices[0] / m_patchStrides[i], patchIndices[1] / m_patchStrides[i]}; diff --git a/unsupported/Eigen/src/Tensor/TensorReduction.h b/unsupported/Eigen/src/Tensor/TensorReduction.h index 1ac2c4b16..faf2be5a7 100644 --- a/unsupported/Eigen/src/Tensor/TensorReduction.h +++ b/unsupported/Eigen/src/Tensor/TensorReduction.h @@ -607,14 +607,13 @@ struct TensorReductionEvaluatorBase 0) { - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_outputStrides[0] = 1; for (int i = 1; i < NumOutputDims; ++i) { m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; m_fastOutputStrides[i] = internal::TensorIntDivisor(m_outputStrides[i]); } - } - else { + } else { m_outputStrides[static_cast(NumOutputDims - 1)] = 1; for (int i = NumOutputDims - 2; i >= 0; --i) { m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; @@ -626,13 +625,12 @@ struct TensorReductionEvaluatorBase 0) { array input_strides; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { input_strides[0] = 1; for (int i = 1; i < NumInputDims; ++i) { input_strides[i] = input_strides[i - 1] * input_dims[i - 1]; } - } - else { + } else { input_strides.back() = 1; for (int i = NumInputDims - 2; i >= 0; --i) { input_strides[i] = input_strides[i + 1] * input_dims[i + 1]; @@ -695,10 +693,9 @@ struct TensorReductionEvaluatorBase= 3)) || (RunningOnSycl)) { bool reducing_inner_dims = true; for (int i = 0; i < NumReducedDims; ++i) { - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { reducing_inner_dims &= m_reduced[i]; - } - else { + } else { reducing_inner_dims &= m_reduced[NumInputDims - 1 - i]; } } @@ -733,10 +730,9 @@ struct TensorReductionEvaluatorBase(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { preserving_inner_dims &= m_reduced[NumInputDims - 1 - i]; - } - else { + } else { preserving_inner_dims &= m_reduced[i]; } } @@ -946,16 +942,15 @@ struct TensorReductionEvaluatorBase(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { return index * m_preservedStrides[0]; - } - else { + } else { return index * m_preservedStrides[NumPreservedStrides - 1]; } } // TBD: optimize the case where we preserve the innermost dimensions. Index startInput = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumOutputDims - 1; i > 0; --i) { // This is index_i in the output tensor. const Index idx = index / m_outputStrides[i]; @@ -968,8 +963,7 @@ struct TensorReductionEvaluatorBase __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) { - EIGEN_IF_CONSTEXPR(sizeof(T) == 4) { + EIGEN_IF_CONSTEXPR (sizeof(T) == 4) { unsigned int oldval = *reinterpret_cast(output); unsigned int newval = oldval; reducer.reduce(accum, reinterpret_cast(&newval)); @@ -42,8 +42,7 @@ __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) return; } } - } - else EIGEN_IF_CONSTEXPR(sizeof(T) == 8) { + } else EIGEN_IF_CONSTEXPR (sizeof(T) == 8) { unsigned long long oldval = *reinterpret_cast(output); unsigned long long newval = oldval; reducer.reduce(accum, reinterpret_cast(&newval)); @@ -59,8 +58,7 @@ __device__ EIGEN_ALWAYS_INLINE void atomicReduce(T* output, T accum, R& reducer) return; } } - } - else { + } else { gpu_assert(0 && "Wordsize not supported"); } } @@ -367,8 +365,9 @@ template __global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void ReductionCleanupKernelHalfFloat(Op reducer, half* output, half* scratch) { eigen_assert(threadIdx.x == 1); typedef packet_traits::type packet_type; - EIGEN_IF_CONSTEXPR(unpacket_traits::size == 1) { *output = *scratch; } - else { + EIGEN_IF_CONSTEXPR (unpacket_traits::size == 1) { + *output = *scratch; + } else { half2* pscratch = reinterpret_cast(scratch); half tmp = __float2half(0.f); for (int i = 0; i < unpacket_traits::size; i += 2) { diff --git a/unsupported/Eigen/src/Tensor/TensorRef.h b/unsupported/Eigen/src/Tensor/TensorRef.h index 5362122f1..c6bb67fe6 100644 --- a/unsupported/Eigen/src/Tensor/TensorRef.h +++ b/unsupported/Eigen/src/Tensor/TensorRef.h @@ -194,13 +194,12 @@ class TensorRefBase : public TensorBase { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(const array& indices) const { const Dimensions& dims = this->dimensions(); Index index = 0; - EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) { index += indices[0]; for (size_t i = 1; i < NumIndices; ++i) { index = index * dims[i] + indices[i]; } - } - else { + } else { index += indices[NumIndices - 1]; for (int i = NumIndices - 2; i >= 0; --i) { index = index * dims[i] + indices[i]; @@ -278,13 +277,12 @@ class TensorRef : public internal::TensorRefBase> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(const array& indices) { const Dimensions& dims = this->dimensions(); Index index = 0; - EIGEN_IF_CONSTEXPR(PlainObjectType::Options & RowMajor) { + EIGEN_IF_CONSTEXPR (PlainObjectType::Options & RowMajor) { index += indices[0]; for (size_t i = 1; i < NumIndices; ++i) { index = index * dims[i] + indices[i]; } - } - else { + } else { index += indices[NumIndices - 1]; for (int i = NumIndices - 2; i >= 0; --i) { index = index * dims[i] + indices[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorReverse.h b/unsupported/Eigen/src/Tensor/TensorReverse.h index dfa566b1d..3c109dca5 100644 --- a/unsupported/Eigen/src/Tensor/TensorReverse.h +++ b/unsupported/Eigen/src/Tensor/TensorReverse.h @@ -116,14 +116,13 @@ struct TensorEvaluator, Device // Compute strides m_dimensions = m_impl.dimensions(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_strides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1]; if (m_strides[i] > 0) m_fastStrides[i] = IndexDivisor(m_strides[i]); } - } - else { + } else { m_strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1]; @@ -151,7 +150,7 @@ struct TensorEvaluator, Device EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index reverseIndex(Index index) const { eigen_assert(index < dimensions().TotalSize()); Index inputIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { Index idx = index / m_fastStrides[i]; @@ -166,8 +165,7 @@ struct TensorEvaluator, Device } else { inputIndex += index; } - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { Index idx = index / m_fastStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorRoll.h b/unsupported/Eigen/src/Tensor/TensorRoll.h index 822c2fc7d..7ef968aee 100644 --- a/unsupported/Eigen/src/Tensor/TensorRoll.h +++ b/unsupported/Eigen/src/Tensor/TensorRoll.h @@ -111,14 +111,13 @@ struct TensorEvaluator, Device> { // Compute strides m_dimensions = m_impl.dimensions(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_strides[0] = 1; for (int i = 1; i < NumDims; ++i) { m_strides[i] = m_strides[i - 1] * m_dimensions[i - 1]; if (m_strides[i] > 0) m_fast_strides[i] = IndexDivisor(m_strides[i]); } - } - else { + } else { m_strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { m_strides[i] = m_strides[i + 1] * m_dimensions[i + 1]; @@ -164,7 +163,7 @@ struct TensorEvaluator, Device> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rollIndex(Index index) const { eigen_assert(index < dimensions().TotalSize()); Index rolledIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { Index idx = index / m_fast_strides[i]; @@ -172,8 +171,7 @@ struct TensorEvaluator, Device> { rolledIndex += roll(idx, m_rolls[i], m_dimensions[i]) * m_strides[i]; } rolledIndex += roll(index, m_rolls[0], m_dimensions[0]); - } - else { + } else { EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { Index idx = index / m_fast_strides[i]; @@ -307,15 +305,14 @@ struct TensorEvaluator, Device> { const Device EIGEN_DEVICE_REF m_device; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void extract_coordinates(Index index, array& coords) const { - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_fast_strides[i]; index -= idx * m_strides[i]; coords[i] = idx; } coords[0] = index; - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_fast_strides[i]; index -= idx * m_strides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorScan.h b/unsupported/Eigen/src/Tensor/TensorScan.h index 75350aeec..78f21e05e 100644 --- a/unsupported/Eigen/src/Tensor/TensorScan.h +++ b/unsupported/Eigen/src/Tensor/TensorScan.h @@ -398,12 +398,11 @@ struct TensorEvaluator, Device> { // Compute stride of scan axis const Dimensions& dims = m_impl.dimensions(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = 0; i < op.axis(); ++i) { m_stride = m_stride * dims[i]; } - } - else { + } else { // dims can only be indexed through unsigned integers, // so use an unsigned type to let the compiler know. // This prevents spurious warnings: "'*((void*)(& evaluator)+64)[18446744073709551615]' may be used uninitialized diff --git a/unsupported/Eigen/src/Tensor/TensorShuffling.h b/unsupported/Eigen/src/Tensor/TensorShuffling.h index 40696ff32..373f5c859 100644 --- a/unsupported/Eigen/src/Tensor/TensorShuffling.h +++ b/unsupported/Eigen/src/Tensor/TensorShuffling.h @@ -120,7 +120,7 @@ struct TensorEvaluator, Device> { } } - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_unshuffledInputStrides[0] = 1; m_outputStrides[0] = 1; @@ -130,8 +130,7 @@ struct TensorEvaluator, Device> { m_fastOutputStrides[i] = internal::TensorIntDivisor(m_outputStrides[i] > 0 ? m_outputStrides[i] : Index(1)); } - } - else { + } else { m_unshuffledInputStrides[NumDims - 1] = 1; m_outputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -265,15 +264,14 @@ struct TensorEvaluator, Device> { const DSizes& output_block_strides, const DSizes, NumDims>& fast_input_block_strides) const { Index output_index = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = input_index / fast_input_block_strides[i]; output_index += idx * output_block_strides[m_inverseShuffle[i]]; input_index -= idx * input_block_strides[i]; } return output_index + input_index * output_block_strides[m_inverseShuffle[0]]; - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = input_index / fast_input_block_strides[i]; output_index += idx * output_block_strides[m_inverseShuffle[i]]; @@ -285,15 +283,14 @@ struct TensorEvaluator, Device> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const { Index inputIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_fastOutputStrides[i]; inputIndex += idx * m_inputStrides[i]; index -= idx * m_outputStrides[i]; } return inputIndex + index * m_inputStrides[0]; - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_fastOutputStrides[i]; inputIndex += idx * m_inputStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorStriding.h b/unsupported/Eigen/src/Tensor/TensorStriding.h index fa82e1eaf..2b6c1eb70 100644 --- a/unsupported/Eigen/src/Tensor/TensorStriding.h +++ b/unsupported/Eigen/src/Tensor/TensorStriding.h @@ -107,7 +107,7 @@ struct TensorEvaluator, Device> { } const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_outputStrides[0] = 1; m_inputStrides[0] = 1; for (int i = 1; i < NumDims; ++i) { @@ -116,8 +116,7 @@ struct TensorEvaluator, Device> { m_inputStrides[i - 1] *= op.strides()[i - 1]; } m_inputStrides[NumDims - 1] *= op.strides()[NumDims - 1]; - } - else { // RowMajor + } else { // RowMajor m_outputStrides[NumDims - 1] = 1; m_inputStrides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { @@ -148,7 +147,7 @@ struct TensorEvaluator, Device> { Index inputIndices[] = {0, 0}; Index indices[] = {index, index + PacketSize - 1}; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / m_outputStrides[i]; @@ -160,8 +159,7 @@ struct TensorEvaluator, Device> { } inputIndices[0] += indices[0] * m_inputStrides[0]; inputIndices[1] += indices[1] * m_inputStrides[0]; - } - else { // RowMajor + } else { // RowMajor EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx0 = indices[0] / m_outputStrides[i]; @@ -208,7 +206,7 @@ struct TensorEvaluator, Device> { protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const { Index inputIndex = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; @@ -216,8 +214,7 @@ struct TensorEvaluator, Device> { index -= idx * m_outputStrides[i]; } inputIndex += index * m_inputStrides[0]; - } - else { // RowMajor + } else { // RowMajor EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStrides[i]; @@ -271,7 +268,7 @@ struct TensorEvaluator, Device> Index inputIndices[] = {0, 0}; Index indices[] = {index, index + PacketSize - 1}; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { EIGEN_UNROLL_LOOP for (int i = NumDims - 1; i > 0; --i) { const Index idx0 = indices[0] / this->m_outputStrides[i]; @@ -283,8 +280,7 @@ struct TensorEvaluator, Device> } inputIndices[0] += indices[0] * this->m_inputStrides[0]; inputIndices[1] += indices[1] * this->m_inputStrides[0]; - } - else { // RowMajor + } else { // RowMajor EIGEN_UNROLL_LOOP for (int i = 0; i < NumDims - 1; ++i) { const Index idx0 = indices[0] / this->m_outputStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorTrace.h b/unsupported/Eigen/src/Tensor/TensorTrace.h index f8953d682..9450ac530 100644 --- a/unsupported/Eigen/src/Tensor/TensorTrace.h +++ b/unsupported/Eigen/src/Tensor/TensorTrace.h @@ -159,13 +159,12 @@ struct TensorEvaluator, Device> { // Compute the output strides if (NumOutputDims > 0) { - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_outputStrides[0] = 1; for (int i = 1; i < NumOutputDims; ++i) { m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; } - } - else { + } else { m_outputStrides.back() = 1; for (int i = NumOutputDims - 2; i >= 0; --i) { m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; @@ -176,13 +175,12 @@ struct TensorEvaluator, Device> { // Compute the input strides if (NumInputDims > 0) { array input_strides; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { input_strides[0] = 1; for (int i = 1; i < NumInputDims; ++i) { input_strides[i] = input_strides[i - 1] * input_dims[i - 1]; } - } - else { + } else { input_strides.back() = 1; for (int i = NumInputDims - 2; i >= 0; --i) { input_strides[i] = input_strides[i + 1] * input_dims[i + 1]; @@ -249,15 +247,14 @@ struct TensorEvaluator, Device> { // Given the output index, finds the first index in the input tensor used to compute the trace EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const { Index startInput = 0; - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumOutputDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; startInput += idx * m_preservedStrides[i]; index -= idx * m_outputStrides[i]; } startInput += index * m_preservedStrides[0]; - } - else { + } else { for (int i = 0; i < NumOutputDims - 1; ++i) { const Index idx = index / m_outputStrides[i]; startInput += idx * m_preservedStrides[i]; diff --git a/unsupported/Eigen/src/Tensor/TensorVolumePatch.h b/unsupported/Eigen/src/Tensor/TensorVolumePatch.h index e55cac38e..d8494047e 100644 --- a/unsupported/Eigen/src/Tensor/TensorVolumePatch.h +++ b/unsupported/Eigen/src/Tensor/TensorVolumePatch.h @@ -209,13 +209,12 @@ struct TensorEvaluator, D const typename TensorEvaluator::Dimensions& input_dims = m_impl.dimensions(); // Cache a few variables. - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_inputDepth = input_dims[0]; m_inputPlanes = input_dims[1]; m_inputRows = input_dims[2]; m_inputCols = input_dims[3]; - } - else { + } else { m_inputDepth = input_dims[NumInputDims - 1]; m_inputPlanes = input_dims[NumInputDims - 2]; m_inputRows = input_dims[NumInputDims - 3]; @@ -288,7 +287,7 @@ struct TensorEvaluator, D eigen_assert(m_outputPlanes > 0); // Dimensions for result of extraction. - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { // ColMajor // 0: depth // 1: patch_planes @@ -304,8 +303,7 @@ struct TensorEvaluator, D for (int i = 5; i < NumDims; ++i) { m_dimensions[i] = input_dims[i - 1]; } - } - else { + } else { // RowMajor // NumDims-1: depth // NumDims-2: patch_planes @@ -324,13 +322,12 @@ struct TensorEvaluator, D } // Strides for the output tensor. - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_rowStride = m_dimensions[1]; m_colStride = m_dimensions[2] * m_rowStride; m_patchStride = m_colStride * m_dimensions[3] * m_dimensions[0]; m_otherStride = m_patchStride * m_dimensions[4]; - } - else { + } else { m_rowStride = m_dimensions[NumDims - 2]; m_colStride = m_dimensions[NumDims - 3] * m_rowStride; m_patchStride = m_colStride * m_dimensions[NumDims - 4] * m_dimensions[NumDims - 1]; @@ -358,10 +355,9 @@ struct TensorEvaluator, D m_fastOutputPlanes = internal::TensorIntDivisor(m_outputPlanes); m_fastOutputPlanesRows = internal::TensorIntDivisor(m_outputPlanesRows); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { m_fastOutputDepth = internal::TensorIntDivisor(m_dimensions[0]); - } - else { + } else { m_fastOutputDepth = internal::TensorIntDivisor(m_dimensions[NumDims - 1]); } } diff --git a/unsupported/test/sparse_extra.cpp b/unsupported/test/sparse_extra.cpp index 93c893771..67a443bc8 100644 --- a/unsupported/test/sparse_extra.cpp +++ b/unsupported/test/sparse_extra.cpp @@ -75,8 +75,8 @@ void sparse_extra(const SparseMatrixType& ref) { // test coeff and coeffRef for (int i = 0; i < (int)zeroCoords.size(); ++i) { VERIFY_IS_MUCH_SMALLER_THAN(m.coeff(zeroCoords[i].x(), zeroCoords[i].y()), eps); - EIGEN_IF_CONSTEXPR((std::is_same >::value)) - VERIFY_RAISES_ASSERT(m.coeffRef(zeroCoords[0].x(), zeroCoords[0].y()) = 5); + EIGEN_IF_CONSTEXPR ((std::is_same >::value)) + VERIFY_RAISES_ASSERT(m.coeffRef(zeroCoords[0].x(), zeroCoords[0].y()) = 5); } VERIFY_IS_APPROX(m, refMat); diff --git a/unsupported/test/tensor_block_access.cpp b/unsupported/test/tensor_block_access.cpp index ed01cd161..7e0dffc7e 100644 --- a/unsupported/test/tensor_block_access.cpp +++ b/unsupported/test/tensor_block_access.cpp @@ -144,15 +144,14 @@ template static Index GetInputIndex(Index output_index, const array& output_to_input_dim_map, const array& input_strides, const array& output_strides) { int input_index = 0; - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = output_index / output_strides[i]; input_index += idx * input_strides[output_to_input_dim_map[i]]; output_index -= idx * output_strides[i]; } return input_index + output_index * input_strides[output_to_input_dim_map[0]]; - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = output_index / output_strides[i]; input_index += idx * input_strides[output_to_input_dim_map[i]]; @@ -165,13 +164,12 @@ static Index GetInputIndex(Index output_index, const array& outp template static array ComputeStrides(const array& sizes) { array strides; - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { strides[0] = 1; for (int i = 1; i < NumDims; ++i) { strides[i] = strides[i - 1] * sizes[i - 1]; } - } - else { + } else { strides[NumDims - 1] = 1; for (int i = NumDims - 2; i >= 0; --i) { strides[i] = strides[i + 1] * sizes[i + 1]; @@ -228,7 +226,7 @@ static void test_uniform_block_shape() { // Test shape 'UniformAllDims' with larger 'max_coeff count' which spills // partially into first inner-most dimension. - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 7 * 5 * 5 * 5 * 5; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -238,8 +236,7 @@ static void test_uniform_block_shape() { VERIFY_IS_EQUAL(5, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } - else { + } else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 5 * 5 * 5 * 5 * 6; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -253,7 +250,7 @@ static void test_uniform_block_shape() { // Test shape 'UniformAllDims' with larger 'max_coeff count' which spills // fully into first inner-most dimension. - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 5 * 5 * 5 * 5; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -263,8 +260,7 @@ static void test_uniform_block_shape() { VERIFY_IS_EQUAL(5, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } - else { + } else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 5 * 5 * 5 * 5 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -278,7 +274,7 @@ static void test_uniform_block_shape() { // Test shape 'UniformAllDims' with larger 'max_coeff count' which spills // fully into first few inner-most dimensions. - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { DSizes dims(7, 5, 6, 17, 7); const Index max_coeff_count = 7 * 5 * 6 * 7 * 5; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -289,8 +285,7 @@ static void test_uniform_block_shape() { VERIFY_IS_EQUAL(7, block.dimensions()[3]); VERIFY_IS_EQUAL(5, block.dimensions()[4]); VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } - else { + } else { DSizes dims(7, 5, 6, 9, 7); const Index max_coeff_count = 5 * 5 * 5 * 6 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -304,7 +299,7 @@ static void test_uniform_block_shape() { } // Test shape 'UniformAllDims' with full allocation to all dims. - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { DSizes dims(7, 5, 6, 17, 7); const Index max_coeff_count = 7 * 5 * 6 * 17 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -315,8 +310,7 @@ static void test_uniform_block_shape() { VERIFY_IS_EQUAL(17, block.dimensions()[3]); VERIFY_IS_EQUAL(7, block.dimensions()[4]); VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } - else { + } else { DSizes dims(7, 5, 6, 9, 7); const Index max_coeff_count = 7 * 5 * 6 * 9 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kUniformAllDims, max_coeff_count, zeroCost()}); @@ -336,7 +330,7 @@ static void test_skewed_inner_dim_block_shape() { typedef internal::TensorBlockMapper<5, Layout> TensorBlockMapper; // Test shape 'SkewedInnerDims' with partial allocation to inner-most dim. - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 10 * 1 * 1 * 1 * 1; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -346,8 +340,7 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(1, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } - else { + } else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 1 * 1 * 1 * 1 * 6; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -360,7 +353,7 @@ static void test_skewed_inner_dim_block_shape() { } // Test shape 'SkewedInnerDims' with full allocation to inner-most dim. - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 1 * 1 * 1 * 1; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -370,8 +363,7 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(1, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } - else { + } else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 1 * 1 * 1 * 1 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -385,7 +377,7 @@ static void test_skewed_inner_dim_block_shape() { // Test shape 'SkewedInnerDims' with full allocation to inner-most dim, // and partial allocation to second inner-dim. - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 3 * 1 * 1 * 1; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -396,8 +388,7 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(1, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } - else { + } else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 1 * 1 * 1 * 15 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -412,7 +403,7 @@ static void test_skewed_inner_dim_block_shape() { // Test shape 'SkewedInnerDims' with full allocation to inner-most dim, // and partial allocation to third inner-dim. - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 5 * 5 * 1 * 1; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -424,8 +415,7 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(1, block.dimensions()[i]); } VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } - else { + } else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 1 * 1 * 5 * 17 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -440,7 +430,7 @@ static void test_skewed_inner_dim_block_shape() { } // Test shape 'SkewedInnerDims' with full allocation to all dims. - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 5 * 6 * 17 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); @@ -451,8 +441,7 @@ static void test_skewed_inner_dim_block_shape() { VERIFY_IS_EQUAL(17, block.dimensions()[3]); VERIFY_IS_EQUAL(7, block.dimensions()[4]); VERIFY(block.dimensions().TotalSize() <= max_coeff_count); - } - else { + } else { DSizes dims(11, 5, 6, 17, 7); const Index max_coeff_count = 11 * 5 * 6 * 17 * 7; TensorBlockMapper block_mapper(dims, {TensorBlockShapeType::kSkewedInnerDims, max_coeff_count, zeroCost()}); diff --git a/unsupported/test/tensor_block_eval.cpp b/unsupported/test/tensor_block_eval.cpp index 6c7acb7c4..bfac872b8 100644 --- a/unsupported/test/tensor_block_eval.cpp +++ b/unsupported/test/tensor_block_eval.cpp @@ -17,10 +17,9 @@ using Eigen::internal::TensorExecutor; template static void setRandomForBinaryProduct(Tensor& tensor) { - EIGEN_IF_CONSTEXPR((std::is_integral::value && !std::is_same::value)) { + EIGEN_IF_CONSTEXPR ((std::is_integral::value && !std::is_same::value)) { setRandomDataInRange(tensor, T(-1000), T(1000)); - } - else { + } else { tensor.setRandom(); } } @@ -86,15 +85,14 @@ static TensorBlockParams SkewedInnerBlock(DSizes dims) // Compute offsets for the first block coefficient. Index index = block.offset(); - EIGEN_IF_CONSTEXPR(static_cast(Layout) == static_cast(ColMajor)) { + EIGEN_IF_CONSTEXPR (static_cast(Layout) == static_cast(ColMajor)) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / strides[i]; index -= idx * strides[i]; offsets[i] = idx; } if (NumDims > 0) offsets[0] = index; - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / strides[i]; index -= idx * strides[i]; diff --git a/unsupported/test/tensor_block_io.cpp b/unsupported/test/tensor_block_io.cpp index 42e65d129..9b596c31c 100644 --- a/unsupported/test/tensor_block_io.cpp +++ b/unsupported/test/tensor_block_io.cpp @@ -38,15 +38,14 @@ template static Index GetInputIndex(Index output_index, const array& output_to_input_dim_map, const array& input_strides, const array& output_strides) { int input_index = 0; - EIGEN_IF_CONSTEXPR(Layout == ColMajor) { + EIGEN_IF_CONSTEXPR (Layout == ColMajor) { for (int i = NumDims - 1; i > 0; --i) { const Index idx = output_index / output_strides[i]; input_index += idx * input_strides[output_to_input_dim_map[i]]; output_index -= idx * output_strides[i]; } return input_index + output_index * input_strides[output_to_input_dim_map[0]]; - } - else { + } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = output_index / output_strides[i]; input_index += idx * input_strides[output_to_input_dim_map[i]];