From 4eecf3c650ea53aa00cae2fe983f95ee6ec6705a Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 3 Jul 2024 14:36:30 -0400 Subject: [PATCH] [SLP]Reorder buildvector/reduction vectorization and fuse the loops. Currently SLP vectorizer tries at first to find reduction nodes, and then vectorize buildvector sequences. Need to try to vectorize wide buildvector sequences at first and only then try to vectorize reductions, and then smaller buildvector sequences. Reviewers: RKSimon Reviewed By: RKSimon Pull Request: https://github.com/llvm/llvm-project/pull/96943 --- .../llvm/Transforms/Vectorize/SLPVectorizer.h | 4 +- .../Transforms/Vectorize/SLPVectorizer.cpp | 32 +++++++++++----- .../AArch64/scalarization-overhead.ll | 37 ++++++++++--------- .../X86/reused-extractelements.ll | 8 ++-- 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h index 3b522dd642db51..95531544a1c817 100644 --- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h +++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h @@ -134,11 +134,11 @@ struct SLPVectorizerPass : public PassInfoMixin { /// Try to vectorize trees that start at insertvalue instructions. bool vectorizeInsertValueInst(InsertValueInst *IVI, BasicBlock *BB, - slpvectorizer::BoUpSLP &R); + slpvectorizer::BoUpSLP &R, bool MaxVFOnly); /// Try to vectorize trees that start at insertelement instructions. bool vectorizeInsertElementInst(InsertElementInst *IEI, BasicBlock *BB, - slpvectorizer::BoUpSLP &R); + slpvectorizer::BoUpSLP &R, bool MaxVFOnly); /// Tries to vectorize \p CmpInts. \Returns true on success. template diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 868e9e2687f57d..11f9ad70dc725c 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -18108,7 +18108,8 @@ bool SLPVectorizerPass::tryToVectorize(ArrayRef Insts, } bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI, - BasicBlock *BB, BoUpSLP &R) { + BasicBlock *BB, BoUpSLP &R, + bool MaxVFOnly) { if (!R.canMapToVector(IVI->getType())) return false; @@ -18119,11 +18120,12 @@ bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI, LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n"); // Aggregate value is unlikely to be processed in vector register. - return tryToVectorizeList(BuildVectorOpds, R); + return tryToVectorizeList(BuildVectorOpds, R, MaxVFOnly); } bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI, - BasicBlock *BB, BoUpSLP &R) { + BasicBlock *BB, BoUpSLP &R, + bool MaxVFOnly) { SmallVector BuildVectorInsts; SmallVector BuildVectorOpds; SmallVector Mask; @@ -18133,7 +18135,7 @@ bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI, return false; LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IEI << "\n"); - return tryToVectorizeList(BuildVectorInsts, R); + return tryToVectorizeList(BuildVectorInsts, R, MaxVFOnly); } template @@ -18353,20 +18355,30 @@ bool SLPVectorizerPass::vectorizeInserts(InstSetVector &Instructions, "This function only accepts Insert instructions"); bool OpsChanged = false; SmallVector PostponedInsts; - // pass1 - try to vectorize reductions only for (auto *I : reverse(Instructions)) { + // pass1 - try to match and vectorize a buildvector sequence for MaxVF only. + if (R.isDeleted(I) || isa(I)) + continue; + if (auto *LastInsertValue = dyn_cast(I)) { + OpsChanged |= + vectorizeInsertValueInst(LastInsertValue, BB, R, /*MaxVFOnly=*/true); + } else if (auto *LastInsertElem = dyn_cast(I)) { + OpsChanged |= + vectorizeInsertElementInst(LastInsertElem, BB, R, /*MaxVFOnly=*/true); + } + // pass2 - try to vectorize reductions only if (R.isDeleted(I)) continue; OpsChanged |= vectorizeHorReduction(nullptr, I, BB, R, TTI, PostponedInsts); - } - // pass2 - try to match and vectorize a buildvector sequence. - for (auto *I : reverse(Instructions)) { if (R.isDeleted(I) || isa(I)) continue; + // pass3 - try to match and vectorize a buildvector sequence. if (auto *LastInsertValue = dyn_cast(I)) { - OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R); + OpsChanged |= + vectorizeInsertValueInst(LastInsertValue, BB, R, /*MaxVFOnly=*/false); } else if (auto *LastInsertElem = dyn_cast(I)) { - OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R); + OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R, + /*MaxVFOnly=*/false); } } // Now try to vectorize postponed instructions. diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/scalarization-overhead.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/scalarization-overhead.ll index c6209fd71063a0..a24cb81541d7c1 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/scalarization-overhead.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/scalarization-overhead.ll @@ -10,32 +10,33 @@ define fastcc i64 @zot(float %arg, float %arg1, float %arg2, float %arg3, float ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[ARG3:%.*]], i32 2 ; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> ; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <4 x float> , [[TMP2]] -; CHECK-NEXT: [[VAL12:%.*]] = fadd fast float [[ARG3]], 1.000000e+00 -; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP2]], float [[VAL12]], i32 0 -; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float 0.000000e+00, i32 1 -; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <4 x float> [[TMP5]], +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> , float [[ARG3]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = fadd fast <2 x float> [[TMP4]], +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP6]], <4 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = fadd fast <4 x float> [[TMP7]], ; CHECK-NEXT: br i1 [[ARG6:%.*]], label [[BB18:%.*]], label [[BB57:%.*]] ; CHECK: bb18: -; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x float> [ [[TMP6]], [[BB:%.*]] ] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x float> [[TMP6]], i32 2 -; CHECK-NEXT: [[VAL23:%.*]] = fmul fast float [[TMP8]], 2.000000e+00 -; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x float> [[TMP6]], i32 3 -; CHECK-NEXT: [[VAL24:%.*]] = fmul fast float [[TMP9]], 3.000000e+00 +; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x float> [ [[TMP8]], [[BB:%.*]] ] +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x float> [[TMP8]], i32 2 +; CHECK-NEXT: [[VAL23:%.*]] = fmul fast float [[TMP10]], 2.000000e+00 +; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x float> [[TMP8]], i32 3 +; CHECK-NEXT: [[VAL24:%.*]] = fmul fast float [[TMP11]], 3.000000e+00 ; CHECK-NEXT: br i1 [[ARG7:%.*]], label [[BB25:%.*]], label [[BB57]] ; CHECK: bb25: -; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x float> [ [[TMP7]], [[BB18]] ] -; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x float> [[TMP3]], i32 1 +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x float> [ [[TMP9]], [[BB18]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x float> [[TMP3]], i32 1 ; CHECK-NEXT: br label [[BB30:%.*]] ; CHECK: bb30: ; CHECK-NEXT: [[VAL31:%.*]] = phi float [ [[VAL55:%.*]], [[BB30]] ], [ 0.000000e+00, [[BB25]] ] -; CHECK-NEXT: [[VAL32:%.*]] = phi float [ [[TMP11]], [[BB30]] ], [ 0.000000e+00, [[BB25]] ] -; CHECK-NEXT: [[TMP12:%.*]] = load <4 x i8>, ptr [[ARG5:%.*]], align 1 -; CHECK-NEXT: [[TMP13:%.*]] = uitofp <4 x i8> [[TMP12]] to <4 x float> -; CHECK-NEXT: [[TMP14:%.*]] = fsub fast <4 x float> [[TMP13]], [[TMP3]] -; CHECK-NEXT: [[TMP15:%.*]] = fmul fast <4 x float> [[TMP14]], [[TMP10]] -; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP15]]) +; CHECK-NEXT: [[VAL32:%.*]] = phi float [ [[TMP13]], [[BB30]] ], [ 0.000000e+00, [[BB25]] ] +; CHECK-NEXT: [[TMP14:%.*]] = load <4 x i8>, ptr [[ARG5:%.*]], align 1 +; CHECK-NEXT: [[TMP15:%.*]] = uitofp <4 x i8> [[TMP14]] to <4 x float> +; CHECK-NEXT: [[TMP16:%.*]] = fsub fast <4 x float> [[TMP15]], [[TMP3]] +; CHECK-NEXT: [[TMP17:%.*]] = fmul fast <4 x float> [[TMP16]], [[TMP12]] +; CHECK-NEXT: [[TMP18:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP17]]) ; CHECK-NEXT: [[VAL55]] = tail call fast float @llvm.minnum.f32(float [[VAL31]], float [[ARG1:%.*]]) -; CHECK-NEXT: [[VAL56:%.*]] = tail call fast float @llvm.maxnum.f32(float [[ARG2:%.*]], float [[TMP16]]) +; CHECK-NEXT: [[VAL56:%.*]] = tail call fast float @llvm.maxnum.f32(float [[ARG2:%.*]], float [[TMP18]]) ; CHECK-NEXT: call void @ham(float [[VAL55]], float [[VAL56]]) ; CHECK-NEXT: br i1 [[ARG8:%.*]], label [[BB30]], label [[BB57]] ; CHECK: bb57: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-extractelements.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-extractelements.ll index 94a1d7aa1951c4..bf4903fd19b09b 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/reused-extractelements.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-extractelements.ll @@ -4,13 +4,11 @@ ; YAML: --- !Missed ; YAML-NEXT: Pass: slp-vectorizer -; YAML-NEXT: Name: NotBeneficial +; YAML-NEXT: Name: NotPossible ; YAML-NEXT: Function: g ; YAML-NEXT: Args: -; YAML-NEXT: - String: 'List vectorization was possible but not beneficial with cost ' -; YAML-NEXT: - Cost: '0' -; YAML-NEXT: - String: ' >= ' -; YAML-NEXT: - Treshold: '0' +; YAML-NEXT: - String: 'Cannot SLP vectorize list: vectorization was impossible' +; YAML-NEXT: - String: ' with available vectorization factors' define <2 x i32> @g(<2 x i32> %x, i32 %a, i32 %b) { ; CHECK-LABEL: @g(