From b7eac8c6fea1ba3930d08011a0e5e3a262bfaece Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 31 Jul 2024 14:21:58 +0800 Subject: [PATCH] [FunctionAttrs] deduce attr `cold` on functions if all CG paths call a `cold` function Closes #101298 --- llvm/lib/Transforms/IPO/FunctionAttrs.cpp | 69 +++ llvm/test/Transforms/FunctionAttrs/cold.ll | 542 ++++++++++++++------- 2 files changed, 433 insertions(+), 178 deletions(-) diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp index d50218aaa3b6cc..603a1565e48c45 100644 --- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp +++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp @@ -82,6 +82,7 @@ STATISTIC(NumNoUnwind, "Number of functions marked as nounwind"); STATISTIC(NumNoFree, "Number of functions marked as nofree"); STATISTIC(NumWillReturn, "Number of functions marked as willreturn"); STATISTIC(NumNoSync, "Number of functions marked as nosync"); +STATISTIC(NumCold, "Number of functions marked as cold"); STATISTIC(NumThinLinkNoRecurse, "Number of functions marked as norecurse during thinlink"); @@ -1745,6 +1746,7 @@ static bool canReturn(Function &F) { return false; } + // Set the noreturn function attribute if possible. static void addNoReturnAttrs(const SCCNodeSet &SCCNodes, SmallSet &Changed) { @@ -1760,6 +1762,72 @@ static void addNoReturnAttrs(const SCCNodeSet &SCCNodes, } } +static bool +allBBPathsGoThroughCold(BasicBlock *BB, + SmallDenseMap &Visited) { + // If BB contains a cold callsite this path through the CG is cold. + // Ignore whether the instructions actually are guranteed to transfer + // execution. Divergent behavior is considered unlikely. + if (any_of(*BB, [](Instruction &I) { + if (auto *CB = dyn_cast(&I)) + return CB->hasFnAttr(Attribute::Cold); + return false; + })) { + Visited[BB] = true; + return true; + } + + auto Succs = successors(BB); + // We found a path that doesn't go through any cold callsite. + if (Succs.empty()) + return false; + + // We didn't find a cold callsite in this BB, so check that all successors + // contain a cold callsite (or that their successors do). + // Potential TODO: We could use static branch hints to assume certain + // successor paths are inherently cold, irrespective of if they contain a cold + // callsite. + for (auto *Succ : Succs) { + // Start with false, this is necessary to ensure we don't turn loops into + // cold. + auto R = Visited.try_emplace(Succ, false); + if (!R.second) { + if (R.first->second) + continue; + return false; + } + if (!allBBPathsGoThroughCold(Succ, Visited)) + return false; + Visited[Succ] = true; + } + + return true; +} + +static bool allPathsGoThroughCold(Function &F) { + SmallDenseMap Visited; + Visited[&F.front()] = false; + return allBBPathsGoThroughCold(&F.front(), Visited); +} + +// Set the cold function attribute if possible. +static void addColdAttrs(const SCCNodeSet &SCCNodes, + SmallSet &Changed) { + for (Function *F : SCCNodes) { + if (!F || !F->hasExactDefinition() || F->hasFnAttribute(Attribute::Naked) || + F->hasFnAttribute(Attribute::Cold) || F->hasFnAttribute(Attribute::Hot)) + continue; + + // Potential TODO: We could add attribute `cold` on functions with `coldcc`. + if (allPathsGoThroughCold(*F)) { + F->addFnAttr(Attribute::Cold); + ++NumCold; + Changed.insert(F); + continue; + } + } +} + static bool functionWillReturn(const Function &F) { // We can infer and propagate function attributes only when we know that the // definition we'll get at link time is *exactly* the definition we see now. @@ -1853,6 +1921,7 @@ deriveAttrsInPostOrder(ArrayRef Functions, AARGetterT &&AARGetter, addArgumentAttrs(Nodes.SCCNodes, Changed); inferConvergent(Nodes.SCCNodes, Changed); addNoReturnAttrs(Nodes.SCCNodes, Changed); + addColdAttrs(Nodes.SCCNodes, Changed); addWillReturn(Nodes.SCCNodes, Changed); addNoUndefAttrs(Nodes.SCCNodes, Changed); diff --git a/llvm/test/Transforms/FunctionAttrs/cold.ll b/llvm/test/Transforms/FunctionAttrs/cold.ll index 1fa8ae06797943..a205fbda062121 100644 --- a/llvm/test/Transforms/FunctionAttrs/cold.ll +++ b/llvm/test/Transforms/FunctionAttrs/cold.ll @@ -54,14 +54,23 @@ while.body2: } define void @test_no_exit() { -; COMMON: Function Attrs: noreturn -; COMMON-LABEL: define void @test_no_exit -; COMMON-SAME: () #[[ATTR2]] { -; COMMON-NEXT: entry: -; COMMON-NEXT: br label [[WHILE_BODY:%.*]] -; COMMON: while.body: -; COMMON-NEXT: call void @cold0() -; COMMON-NEXT: br label [[WHILE_BODY]] +; FNATTRS: Function Attrs: cold noreturn +; FNATTRS-LABEL: define void @test_no_exit +; FNATTRS-SAME: () #[[ATTR3:[0-9]+]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: br label [[WHILE_BODY:%.*]] +; FNATTRS: while.body: +; FNATTRS-NEXT: call void @cold0() +; FNATTRS-NEXT: br label [[WHILE_BODY]] +; +; ATTRIBUTOR: Function Attrs: noreturn +; ATTRIBUTOR-LABEL: define void @test_no_exit +; ATTRIBUTOR-SAME: () #[[ATTR2]] { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY:%.*]] +; ATTRIBUTOR: while.body: +; ATTRIBUTOR-NEXT: call void @cold0() +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY]] ; entry: br label %while.body @@ -72,17 +81,29 @@ while.body: } define void @test_no_exit2() { -; COMMON: Function Attrs: noreturn -; COMMON-LABEL: define void @test_no_exit2 -; COMMON-SAME: () #[[ATTR2]] { -; COMMON-NEXT: entry: -; COMMON-NEXT: br label [[WHILE_BODY:%.*]] -; COMMON: while.body: -; COMMON-NEXT: call void @not_cold0() -; COMMON-NEXT: br label [[WHILE_BODY2:%.*]] -; COMMON: while.body2: -; COMMON-NEXT: call void @cold1() -; COMMON-NEXT: br label [[WHILE_BODY]] +; FNATTRS: Function Attrs: cold noreturn +; FNATTRS-LABEL: define void @test_no_exit2 +; FNATTRS-SAME: () #[[ATTR3]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: br label [[WHILE_BODY:%.*]] +; FNATTRS: while.body: +; FNATTRS-NEXT: call void @not_cold0() +; FNATTRS-NEXT: br label [[WHILE_BODY2:%.*]] +; FNATTRS: while.body2: +; FNATTRS-NEXT: call void @cold1() +; FNATTRS-NEXT: br label [[WHILE_BODY]] +; +; ATTRIBUTOR: Function Attrs: noreturn +; ATTRIBUTOR-LABEL: define void @test_no_exit2 +; ATTRIBUTOR-SAME: () #[[ATTR2]] { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY:%.*]] +; ATTRIBUTOR: while.body: +; ATTRIBUTOR-NEXT: call void @not_cold0() +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY2:%.*]] +; ATTRIBUTOR: while.body2: +; ATTRIBUTOR-NEXT: call void @cold1() +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY]] ; entry: br label %while.body @@ -97,18 +118,32 @@ while.body2: } define dso_local void @test_entry(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_entry -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: br label [[IF_END]] -; COMMON: if.end: -; COMMON-NEXT: tail call void @not_cold1() -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_entry +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0:[0-9]+]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: br label [[IF_END]] +; FNATTRS: if.end: +; FNATTRS-NEXT: tail call void @not_cold1() +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_entry +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END]] +; ATTRIBUTOR: if.end: +; ATTRIBUTOR-NEXT: tail call void @not_cold1() +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @cold0() @@ -125,12 +160,19 @@ if.end: } define dso_local void @test_hot_fail(i32 noundef %x) hot { -; COMMON: Function Attrs: hot -; COMMON-LABEL: define dso_local void @test_hot_fail -; COMMON-SAME: (i32 noundef [[X:%.*]]) #[[ATTR3:[0-9]+]] { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: hot +; FNATTRS-LABEL: define dso_local void @test_hot_fail +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR4:[0-9]+]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR: Function Attrs: hot +; ATTRIBUTOR-LABEL: define dso_local void @test_hot_fail +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) #[[ATTR3:[0-9]+]] { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @cold0() @@ -138,19 +180,34 @@ entry: } define dso_local void @test_br2(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_br2 -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END:%.*]] -; COMMON: if.else: -; COMMON-NEXT: tail call void @cold1() -; COMMON-NEXT: br label [[IF_END]] -; COMMON: if.end: -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_br2 +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END:%.*]] +; FNATTRS: if.else: +; FNATTRS-NEXT: tail call void @cold1() +; FNATTRS-NEXT: br label [[IF_END]] +; FNATTRS: if.end: +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_br2 +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END:%.*]] +; ATTRIBUTOR: if.else: +; ATTRIBUTOR-NEXT: tail call void @cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END]] +; ATTRIBUTOR: if.end: +; ATTRIBUTOR-NEXT: ret void ; entry: %tobool.not = icmp eq i32 %x, 0 @@ -169,21 +226,38 @@ if.end: } define dso_local void @test_exit(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_exit -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: tail call void @not_cold1() -; COMMON-NEXT: br label [[IF_END:%.*]] -; COMMON: if.else: -; COMMON-NEXT: tail call void @not_cold2() -; COMMON-NEXT: br label [[IF_END]] -; COMMON: if.end: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_exit +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: tail call void @not_cold1() +; FNATTRS-NEXT: br label [[IF_END:%.*]] +; FNATTRS: if.else: +; FNATTRS-NEXT: tail call void @not_cold2() +; FNATTRS-NEXT: br label [[IF_END]] +; FNATTRS: if.end: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_exit +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: tail call void @not_cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END:%.*]] +; ATTRIBUTOR: if.else: +; ATTRIBUTOR-NEXT: tail call void @not_cold2() +; ATTRIBUTOR-NEXT: br label [[IF_END]] +; ATTRIBUTOR: if.end: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @not_cold0() @@ -204,54 +278,104 @@ if.end: } define dso_local void @test_complex(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_complex -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: [[CALL:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL]], 0 -; COMMON-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] -; COMMON: if.then2: -; COMMON-NEXT: tail call void @cold1() -; COMMON-NEXT: br label [[IF_END12:%.*]] -; COMMON: if.else: -; COMMON-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 -; COMMON-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] -; COMMON: if.then5: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END12]] -; COMMON: if.else6: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ -; COMMON-NEXT: i32 0, label [[SW_BB:%.*]] -; COMMON-NEXT: i32 1, label [[SW_BB8:%.*]] -; COMMON-NEXT: i32 2, label [[SW_BB9:%.*]] -; COMMON-NEXT: ] -; COMMON: sw.bb: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: br label [[CALL_COLD:%.*]] -; COMMON: sw.bb8: -; COMMON-NEXT: tail call void @not_cold1() -; COMMON-NEXT: br label [[CALL_COLD]] -; COMMON: sw.bb9: -; COMMON-NEXT: tail call void @not_cold2() -; COMMON-NEXT: br label [[CALL_COLD]] -; COMMON: sw.default: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END12]] -; COMMON: call_cold: -; COMMON-NEXT: tail call void @cold_at_cb() #[[ATTR0:[0-9]+]] -; COMMON-NEXT: br label [[IF_END12]] -; COMMON: if.else11: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END12]] -; COMMON: if.end12: -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_complex +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: [[CALL:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] +; FNATTRS: if.then2: +; FNATTRS-NEXT: tail call void @cold1() +; FNATTRS-NEXT: br label [[IF_END12:%.*]] +; FNATTRS: if.else: +; FNATTRS-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] +; FNATTRS: if.then5: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END12]] +; FNATTRS: if.else6: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ +; FNATTRS-NEXT: i32 0, label [[SW_BB:%.*]] +; FNATTRS-NEXT: i32 1, label [[SW_BB8:%.*]] +; FNATTRS-NEXT: i32 2, label [[SW_BB9:%.*]] +; FNATTRS-NEXT: ] +; FNATTRS: sw.bb: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: br label [[CALL_COLD:%.*]] +; FNATTRS: sw.bb8: +; FNATTRS-NEXT: tail call void @not_cold1() +; FNATTRS-NEXT: br label [[CALL_COLD]] +; FNATTRS: sw.bb9: +; FNATTRS-NEXT: tail call void @not_cold2() +; FNATTRS-NEXT: br label [[CALL_COLD]] +; FNATTRS: sw.default: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END12]] +; FNATTRS: call_cold: +; FNATTRS-NEXT: tail call void @cold_at_cb() #[[ATTR0]] +; FNATTRS-NEXT: br label [[IF_END12]] +; FNATTRS: if.else11: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END12]] +; FNATTRS: if.end12: +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_complex +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: [[CALL:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] +; ATTRIBUTOR: if.then2: +; ATTRIBUTOR-NEXT: tail call void @cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END12:%.*]] +; ATTRIBUTOR: if.else: +; ATTRIBUTOR-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] +; ATTRIBUTOR: if.then5: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END12]] +; ATTRIBUTOR: if.else6: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ +; ATTRIBUTOR-NEXT: i32 0, label [[SW_BB:%.*]] +; ATTRIBUTOR-NEXT: i32 1, label [[SW_BB8:%.*]] +; ATTRIBUTOR-NEXT: i32 2, label [[SW_BB9:%.*]] +; ATTRIBUTOR-NEXT: ] +; ATTRIBUTOR: sw.bb: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD:%.*]] +; ATTRIBUTOR: sw.bb8: +; ATTRIBUTOR-NEXT: tail call void @not_cold1() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD]] +; ATTRIBUTOR: sw.bb9: +; ATTRIBUTOR-NEXT: tail call void @not_cold2() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD]] +; ATTRIBUTOR: sw.default: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END12]] +; ATTRIBUTOR: call_cold: +; ATTRIBUTOR-NEXT: tail call void @cold_at_cb() #[[ATTR0:[0-9]+]] +; ATTRIBUTOR-NEXT: br label [[IF_END12]] +; ATTRIBUTOR: if.else11: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END12]] +; ATTRIBUTOR: if.end12: +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @not_cold0() @@ -314,63 +438,122 @@ if.end12: } define dso_local void @test_complex2(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_complex2 -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: [[CALL12:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL12]], 0 -; COMMON-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] -; COMMON: if.then2: -; COMMON-NEXT: tail call void @cold1() -; COMMON-NEXT: br label [[IF_END16:%.*]] -; COMMON: if.else: -; COMMON-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 -; COMMON-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] -; COMMON: if.then5: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END16]] -; COMMON: if.else6: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ -; COMMON-NEXT: i32 0, label [[SW_BB:%.*]] -; COMMON-NEXT: i32 1, label [[SW_BB8:%.*]] -; COMMON-NEXT: i32 2, label [[SW_BB9:%.*]] -; COMMON-NEXT: ] -; COMMON: sw.bb: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: br label [[CALL_COLD:%.*]] -; COMMON: sw.bb8: -; COMMON-NEXT: tail call void @not_cold1() -; COMMON-NEXT: br label [[CALL_COLD]] -; COMMON: sw.bb9: -; COMMON-NEXT: tail call void @not_cold2() -; COMMON-NEXT: br label [[CALL_COLD]] -; COMMON: sw.default: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END16]] -; COMMON: call_cold: -; COMMON-NEXT: tail call void @cold_at_cb() #[[ATTR0]] -; COMMON-NEXT: br label [[IF_END16]] -; COMMON: if.else11: -; COMMON-NEXT: [[CMP:%.*]] = icmp slt i32 [[CALL12]], 1 -; COMMON-NEXT: br i1 [[CMP]], label [[IF_END14:%.*]], label [[FOR_BODY:%.*]] -; COMMON: if.end14: -; COMMON-NEXT: tail call void @cold1() -; COMMON-NEXT: br label [[IF_END16]] -; COMMON: for.body: -; COMMON-NEXT: [[I_021:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[IF_ELSE11]] ] -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: [[INC]] = add nuw nsw i32 [[I_021]], 1 -; COMMON-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[CALL12]] -; COMMON-NEXT: br i1 [[EXITCOND_NOT]], label [[IF_END16]], label [[FOR_BODY]] -; COMMON: if.end16: -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_complex2 +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: [[CALL12:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL12]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] +; FNATTRS: if.then2: +; FNATTRS-NEXT: tail call void @cold1() +; FNATTRS-NEXT: br label [[IF_END16:%.*]] +; FNATTRS: if.else: +; FNATTRS-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] +; FNATTRS: if.then5: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END16]] +; FNATTRS: if.else6: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ +; FNATTRS-NEXT: i32 0, label [[SW_BB:%.*]] +; FNATTRS-NEXT: i32 1, label [[SW_BB8:%.*]] +; FNATTRS-NEXT: i32 2, label [[SW_BB9:%.*]] +; FNATTRS-NEXT: ] +; FNATTRS: sw.bb: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: br label [[CALL_COLD:%.*]] +; FNATTRS: sw.bb8: +; FNATTRS-NEXT: tail call void @not_cold1() +; FNATTRS-NEXT: br label [[CALL_COLD]] +; FNATTRS: sw.bb9: +; FNATTRS-NEXT: tail call void @not_cold2() +; FNATTRS-NEXT: br label [[CALL_COLD]] +; FNATTRS: sw.default: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END16]] +; FNATTRS: call_cold: +; FNATTRS-NEXT: tail call void @cold_at_cb() #[[ATTR0]] +; FNATTRS-NEXT: br label [[IF_END16]] +; FNATTRS: if.else11: +; FNATTRS-NEXT: [[CMP:%.*]] = icmp slt i32 [[CALL12]], 1 +; FNATTRS-NEXT: br i1 [[CMP]], label [[IF_END14:%.*]], label [[FOR_BODY:%.*]] +; FNATTRS: if.end14: +; FNATTRS-NEXT: tail call void @cold1() +; FNATTRS-NEXT: br label [[IF_END16]] +; FNATTRS: for.body: +; FNATTRS-NEXT: [[I_021:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[IF_ELSE11]] ] +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: [[INC]] = add nuw nsw i32 [[I_021]], 1 +; FNATTRS-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[CALL12]] +; FNATTRS-NEXT: br i1 [[EXITCOND_NOT]], label [[IF_END16]], label [[FOR_BODY]] +; FNATTRS: if.end16: +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_complex2 +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: [[CALL12:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL12]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] +; ATTRIBUTOR: if.then2: +; ATTRIBUTOR-NEXT: tail call void @cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END16:%.*]] +; ATTRIBUTOR: if.else: +; ATTRIBUTOR-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] +; ATTRIBUTOR: if.then5: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END16]] +; ATTRIBUTOR: if.else6: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ +; ATTRIBUTOR-NEXT: i32 0, label [[SW_BB:%.*]] +; ATTRIBUTOR-NEXT: i32 1, label [[SW_BB8:%.*]] +; ATTRIBUTOR-NEXT: i32 2, label [[SW_BB9:%.*]] +; ATTRIBUTOR-NEXT: ] +; ATTRIBUTOR: sw.bb: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD:%.*]] +; ATTRIBUTOR: sw.bb8: +; ATTRIBUTOR-NEXT: tail call void @not_cold1() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD]] +; ATTRIBUTOR: sw.bb9: +; ATTRIBUTOR-NEXT: tail call void @not_cold2() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD]] +; ATTRIBUTOR: sw.default: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END16]] +; ATTRIBUTOR: call_cold: +; ATTRIBUTOR-NEXT: tail call void @cold_at_cb() #[[ATTR0]] +; ATTRIBUTOR-NEXT: br label [[IF_END16]] +; ATTRIBUTOR: if.else11: +; ATTRIBUTOR-NEXT: [[CMP:%.*]] = icmp slt i32 [[CALL12]], 1 +; ATTRIBUTOR-NEXT: br i1 [[CMP]], label [[IF_END14:%.*]], label [[FOR_BODY:%.*]] +; ATTRIBUTOR: if.end14: +; ATTRIBUTOR-NEXT: tail call void @cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END16]] +; ATTRIBUTOR: for.body: +; ATTRIBUTOR-NEXT: [[I_021:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[IF_ELSE11]] ] +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: [[INC]] = add nuw nsw i32 [[I_021]], 1 +; ATTRIBUTOR-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[CALL12]] +; ATTRIBUTOR-NEXT: br i1 [[EXITCOND_NOT]], label [[IF_END16]], label [[FOR_BODY]] +; ATTRIBUTOR: if.end16: +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @not_cold0() @@ -485,7 +668,7 @@ define dso_local void @test_complex_fail(i32 noundef %x) { ; COMMON-NEXT: tail call void @cold0() ; COMMON-NEXT: br label [[IF_END12]] ; COMMON: call_cold: -; COMMON-NEXT: tail call void @cold_at_cb() #[[ATTR0]] +; COMMON-NEXT: tail call void @cold_at_cb() #[[ATTR0:[0-9]+]] ; COMMON-NEXT: br label [[IF_END12]] ; COMMON: if.else11: ; COMMON-NEXT: tail call void @cold0() @@ -684,11 +867,14 @@ if.end16: } ;. -; COMMON: attributes #[[ATTR0]] = { cold } -; COMMON: attributes #[[ATTR1]] = { nofree norecurse noreturn nosync nounwind memory(none) } -; COMMON: attributes #[[ATTR2]] = { noreturn } -; COMMON: attributes #[[ATTR3]] = { hot } +; FNATTRS: attributes #[[ATTR0]] = { cold } +; FNATTRS: attributes #[[ATTR1]] = { nofree norecurse noreturn nosync nounwind memory(none) } +; FNATTRS: attributes #[[ATTR2]] = { noreturn } +; FNATTRS: attributes #[[ATTR3]] = { cold noreturn } +; FNATTRS: attributes #[[ATTR4]] = { hot } +;. +; ATTRIBUTOR: attributes #[[ATTR0]] = { cold } +; ATTRIBUTOR: attributes #[[ATTR1]] = { nofree norecurse noreturn nosync nounwind memory(none) } +; ATTRIBUTOR: attributes #[[ATTR2]] = { noreturn } +; ATTRIBUTOR: attributes #[[ATTR3]] = { hot } ;. -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; ATTRIBUTOR: {{.*}} -; FNATTRS: {{.*}}