Skip to content

Commit

Permalink
Merge branch 'llvm:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
xingxue-ibm authored Apr 17, 2024
2 parents 68c3976 + 421a8c5 commit 302b83d
Show file tree
Hide file tree
Showing 3,109 changed files with 132,823 additions and 82,792 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
4 changes: 4 additions & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@ clang/test/AST/Interp/ @tbaederr
/mlir/**/LLVMIR/**/BasicPtxBuilderInterface* @grypp
/mlir/**/NVVM*/ @grypp

# MLIR Python Bindings
/mlir/test/python/ @makslevental @stellaraccident
/mlir/python/ @makslevental @stellaraccident

# BOLT
/bolt/ @aaupov @maksfb @rafaelauler @ayermolo @dcci

Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/pr-code-format.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
name: "Check code formatting"

permissions:
contents: read

on:
pull_request:
branches:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release-lit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ jobs:
cd llvm/utils/lit
# Remove 'dev' suffix from lit version.
sed -i 's/ + "dev"//g' lit/__init__.py
python3 setup.py sdist
python3 setup.py sdist bdist_wheel
- name: Upload lit to test.pypi.org
uses: pypa/gh-action-pypi-publish@release/v1
Expand Down
9 changes: 5 additions & 4 deletions bolt/docs/BAT.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,10 @@ Hot indices are delta encoded, implicitly starting at zero.
| `FuncHash` | 8b | Function hash for input function | Hot |
| `NumBlocks` | ULEB128 | Number of basic blocks in the original function | Hot |
| `NumSecEntryPoints` | ULEB128 | Number of secondary entry points in the original function | Hot |
| `ColdInputSkew` | ULEB128 | Skew to apply to all input offsets | Cold |
| `NumEntries` | ULEB128 | Number of address translation entries for a function | Both |
| `EqualElems` | ULEB128 | Number of equal offsets in the beginning of a function | Hot |
| `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | If `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit | Hot |
| `EqualElems` | ULEB128 | Number of equal offsets in the beginning of a function | Both |
| `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | If `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit | Both |

Function header is followed by *Address Translation Table* with `NumEntries`
total entries, and *Secondary Entry Points* table with `NumSecEntryPoints`
Expand All @@ -99,8 +100,8 @@ entry is encoded. Input offsets implicitly start at zero.
| `BBHash` | Optional, 8b | Basic block hash in input binary | BB |
| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB |

For hot fragments, the table omits the first `EqualElems` input offsets
where the input offset equals output offset.
The table omits the first `EqualElems` input offsets where the input offset
equals output offset.

`BRANCHENTRY` bit denotes whether a given offset pair is a control flow source
(branch or call instruction). If not set, it signifies a control flow target
Expand Down
1 change: 0 additions & 1 deletion bolt/include/bolt/Core/BinaryData.h
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ class BinaryData {
std::vector<MCSymbol *> &getSymbols() { return Symbols; }

bool hasName(StringRef Name) const;
bool hasNameRegex(StringRef Name) const;
bool nameStartsWith(StringRef Prefix) const;

bool hasSymbol(const MCSymbol *Symbol) const {
Expand Down
3 changes: 2 additions & 1 deletion bolt/include/bolt/Core/BinaryFunction.h
Original file line number Diff line number Diff line change
Expand Up @@ -1402,7 +1402,8 @@ class BinaryFunction {

/// Return true if the function has CFI instructions
bool hasCFI() const {
return !FrameInstructions.empty() || !CIEFrameInstructions.empty();
return !FrameInstructions.empty() || !CIEFrameInstructions.empty() ||
IsInjected;
}

/// Return unique number associated with the function.
Expand Down
17 changes: 8 additions & 9 deletions bolt/include/bolt/Profile/BoltAddressTranslation.h
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,6 @@ class BoltAddressTranslation {
/// True if a given \p Address is a function with translation table entry.
bool isBATFunction(uint64_t Address) const { return Maps.count(Address); }

/// Returns branch offsets grouped by containing basic block in a given
/// function.
std::unordered_map<uint32_t, std::vector<uint32_t>>
getBFBranches(uint64_t FuncOutputAddress) const;

/// For a given \p Symbol in the output binary and known \p InputOffset
/// return a corresponding pair of parent BinaryFunction and secondary entry
/// point in it.
Expand Down Expand Up @@ -154,9 +149,9 @@ class BoltAddressTranslation {
/// entries in function address translation map.
APInt calculateBranchEntriesBitMask(MapTy &Map, size_t EqualElems);

/// Calculate the number of equal offsets (output = input) in the beginning
/// of the function.
size_t getNumEqualOffsets(const MapTy &Map) const;
/// Calculate the number of equal offsets (output = input - skew) in the
/// beginning of the function.
size_t getNumEqualOffsets(const MapTy &Map, uint32_t Skew) const;

std::map<uint64_t, MapTy> Maps;

Expand Down Expand Up @@ -193,7 +188,7 @@ class BoltAddressTranslation {
EntryTy(unsigned Index, size_t Hash) : Index(Index), Hash(Hash) {}
};

std::unordered_map<uint32_t, EntryTy> Map;
std::map<uint32_t, EntryTy> Map;
const EntryTy &getEntry(uint32_t BBInputOffset) const {
auto It = Map.find(BBInputOffset);
assert(It != Map.end());
Expand All @@ -218,6 +213,10 @@ class BoltAddressTranslation {
}

size_t getNumBasicBlocks() const { return Map.size(); }

auto begin() const { return Map.begin(); }
auto end() const { return Map.end(); }
auto upper_bound(uint32_t Offset) const { return Map.upper_bound(Offset); }
};

/// Map function output address to its hash and basic blocks hash map.
Expand Down
19 changes: 9 additions & 10 deletions bolt/include/bolt/Profile/DataAggregator.h
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,10 @@ class DataAggregator : public DataReader {
/// Aggregation statistics
uint64_t NumInvalidTraces{0};
uint64_t NumLongRangeTraces{0};
/// Specifies how many samples were recorded in cold areas if we are dealing
/// with profiling data collected in a bolted binary. For LBRs, incremented
/// for the source of the branch to avoid counting cold activity twice (one
/// for source and another for destination).
uint64_t NumColdSamples{0};

/// Looks into system PATH for Linux Perf and set up the aggregator to use it
Expand All @@ -245,14 +249,12 @@ class DataAggregator : public DataReader {
/// disassembled BinaryFunctions
BinaryFunction *getBinaryFunctionContainingAddress(uint64_t Address) const;

/// Perform BAT translation for a given \p Func and return the parent
/// BinaryFunction or nullptr.
BinaryFunction *getBATParentFunction(const BinaryFunction &Func) const;

/// Retrieve the location name to be used for samples recorded in \p Func.
/// If doing BAT translation, link cold parts to the hot part names (used by
/// the original binary). \p Count specifies how many samples were recorded
/// at that location, so we can tally total activity in cold areas if we are
/// dealing with profiling data collected in a bolted binary. For LBRs,
/// \p Count should only be used for the source of the branch to avoid
/// counting cold activity twice (one for source and another for destination).
StringRef getLocationName(BinaryFunction &Func, uint64_t Count);
StringRef getLocationName(const BinaryFunction &Func) const;

/// Semantic actions - parser hooks to interpret parsed perf samples
/// Register a sample (non-LBR mode), i.e. a new hit at \p Address
Expand Down Expand Up @@ -467,9 +469,6 @@ class DataAggregator : public DataReader {
std::error_code writeBATYAML(BinaryContext &BC,
StringRef OutputFilename) const;

/// Fixup profile collected on BOLTed binary, namely handle split functions.
void fixupBATProfile(BinaryContext &BC);

/// Filter out binaries based on PID
void filterBinaryMMapInfo();

Expand Down
10 changes: 0 additions & 10 deletions bolt/include/bolt/Rewrite/RewriteInstance.h
Original file line number Diff line number Diff line change
Expand Up @@ -368,13 +368,6 @@ class RewriteInstance {
/// rewritten binary.
void patchBuildID();

/// Return file offset corresponding to a given virtual address.
uint64_t getFileOffsetFor(uint64_t Address) {
assert(Address >= NewTextSegmentAddress &&
"address in not in the new text segment");
return Address - NewTextSegmentAddress + NewTextSegmentOffset;
}

/// Return file offset corresponding to a virtual \p Address.
/// Return 0 if the address has no mapping in the file, including being
/// part of .bss section.
Expand All @@ -398,9 +391,6 @@ class RewriteInstance {
/// Return true if the section holds debug information.
static bool isDebugSection(StringRef SectionName);

/// Return true if the section holds linux kernel symbol information.
static bool isKSymtabSection(StringRef SectionName);

/// Adds Debug section to overwrite.
static void addToDebugSectionsToOverwrite(const char *Section) {
DebugSectionsToOverwrite.emplace_back(Section);
Expand Down
28 changes: 22 additions & 6 deletions bolt/lib/Core/BinaryContext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,9 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
const uint64_t NextJTAddress,
JumpTable::AddressesType *EntriesAsAddress,
bool *HasEntryInFragment) const {
// Target address of __builtin_unreachable.
const uint64_t UnreachableAddress = BF.getAddress() + BF.getSize();

// Is one of the targets __builtin_unreachable?
bool HasUnreachable = false;

Expand All @@ -564,9 +567,15 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
// Number of targets other than __builtin_unreachable.
uint64_t NumRealEntries = 0;

auto addEntryAddress = [&](uint64_t EntryAddress) {
if (EntriesAsAddress)
EntriesAsAddress->emplace_back(EntryAddress);
// Size of the jump table without trailing __builtin_unreachable entries.
size_t TrimmedSize = 0;

auto addEntryAddress = [&](uint64_t EntryAddress, bool Unreachable = false) {
if (!EntriesAsAddress)
return;
EntriesAsAddress->emplace_back(EntryAddress);
if (!Unreachable)
TrimmedSize = EntriesAsAddress->size();
};

ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);
Expand Down Expand Up @@ -618,8 +627,8 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
: *getPointerAtAddress(EntryAddress);

// __builtin_unreachable() case.
if (Value == BF.getAddress() + BF.getSize()) {
addEntryAddress(Value);
if (Value == UnreachableAddress) {
addEntryAddress(Value, /*Unreachable*/ true);
HasUnreachable = true;
LLVM_DEBUG(dbgs() << formatv("OK: {0:x} __builtin_unreachable\n", Value));
continue;
Expand Down Expand Up @@ -673,6 +682,13 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
addEntryAddress(Value);
}

// Trim direct/normal jump table to exclude trailing unreachable entries that
// can collide with a function address.
if (Type == JumpTable::JTT_NORMAL && EntriesAsAddress &&
TrimmedSize != EntriesAsAddress->size() &&
getBinaryFunctionAtAddress(UnreachableAddress))
EntriesAsAddress->resize(TrimmedSize);

// It's a jump table if the number of real entries is more than 1, or there's
// one real entry and one or more special targets. If there are only multiple
// special targets, then it's not a jump table.
Expand Down Expand Up @@ -1864,7 +1880,7 @@ MarkerSymType BinaryContext::getMarkerType(const SymbolRef &Symbol) const {
// For aarch64 and riscv, the ABI defines mapping symbols so we identify data
// in the code section (see IHI0056B). $x identifies a symbol starting code or
// the end of a data chunk inside code, $d identifies start of data.
if ((!isAArch64() && !isRISCV()) || ELFSymbolRef(Symbol).getSize())
if (isX86() || ELFSymbolRef(Symbol).getSize())
return MarkerSymType::NONE;

Expected<StringRef> NameOrError = Symbol.getName();
Expand Down
8 changes: 0 additions & 8 deletions bolt/lib/Core/BinaryData.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,6 @@ bool BinaryData::hasName(StringRef Name) const {
return false;
}

bool BinaryData::hasNameRegex(StringRef NameRegex) const {
Regex MatchName(NameRegex);
for (const MCSymbol *Symbol : Symbols)
if (MatchName.match(Symbol->getName()))
return true;
return false;
}

bool BinaryData::nameStartsWith(StringRef Prefix) const {
for (const MCSymbol *Symbol : Symbols)
if (Symbol->getName().starts_with(Prefix))
Expand Down
2 changes: 1 addition & 1 deletion bolt/lib/Core/BinaryEmitter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,

// Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86.
// This is a workaround for invalid NOPs handling by asm/disasm layer.
if (BC.MIB->isNoop(Instr) && BC.isX86()) {
if (BC.isX86() && BC.MIB->isNoop(Instr)) {
if (std::optional<uint32_t> Size = BC.MIB->getSize(Instr)) {
SmallString<15> Code;
raw_svector_ostream VecOS(Code);
Expand Down
22 changes: 13 additions & 9 deletions bolt/lib/Core/Relocation.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1064,21 +1064,19 @@ MCBinaryExpr::Opcode Relocation::getComposeOpcodeFor(uint64_t Type) {
}
}

#define ELF_RELOC(name, value) #name,

void Relocation::print(raw_ostream &OS) const {
static const char *X86RelocNames[] = {
#include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
};
static const char *AArch64RelocNames[] = {
#include "llvm/BinaryFormat/ELFRelocs/AArch64.def"
};
switch (Arch) {
default:
OS << "RType:" << Twine::utohexstr(Type);
break;

case Triple::aarch64:
static const char *const AArch64RelocNames[] = {
#define ELF_RELOC(name, value) #name,
#include "llvm/BinaryFormat/ELFRelocs/AArch64.def"
#undef ELF_RELOC
};
assert(Type < ArrayRef(AArch64RelocNames).size());
OS << AArch64RelocNames[Type];
break;

Expand All @@ -1088,16 +1086,22 @@ void Relocation::print(raw_ostream &OS) const {
switch (Type) {
default:
llvm_unreachable("illegal RISC-V relocation");
#undef ELF_RELOC
#define ELF_RELOC(name, value) \
case value: \
OS << #name; \
break;
#include "llvm/BinaryFormat/ELFRelocs/RISCV.def"
#undef ELF_RELOC
}
break;

case Triple::x86_64:
static const char *const X86RelocNames[] = {
#define ELF_RELOC(name, value) #name,
#include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
#undef ELF_RELOC
};
assert(Type < ArrayRef(X86RelocNames).size());
OS << X86RelocNames[Type];
break;
}
Expand Down
Loading

0 comments on commit 302b83d

Please sign in to comment.