Skip to content

Commit

Permalink
Merge branch 'main' into format3
Browse files Browse the repository at this point in the history
  • Loading branch information
ilya-biryukov authored Jan 15, 2024
2 parents 8060996 + dc01b59 commit 18d06c2
Show file tree
Hide file tree
Showing 3,904 changed files with 157,222 additions and 70,869 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
2 changes: 1 addition & 1 deletion .ci/monolithic-linux.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
-D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \
-D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
-D LLVM_ENABLE_LLD=ON \
-D CMAKE_CXX_FLAGS=-gmlt \
-D BOLT_CLANG_EXE=/usr/bin/clang \
Expand Down
2 changes: 1 addition & 1 deletion .ci/monolithic-windows.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
-D LLVM_ENABLE_ASSERTIONS=ON \
-D LLVM_BUILD_EXAMPLES=ON \
-D COMPILER_RT_BUILD_LIBFUZZER=OFF \
-D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \
-D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
-D COMPILER_RT_BUILD_ORC=OFF \
-D CMAKE_C_COMPILER_LAUNCHER=sccache \
-D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/README.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
Github action workflows should be stored in this directrory.
Github action workflows should be stored in this directory.
60 changes: 60 additions & 0 deletions .github/workflows/build-ci-container.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@

name: Build CI Container

permissions:
contents: read

on:
push:
branches:
- main
paths:
- .github/workflows/build-ci-container.yml
- '.github/workflows/containers/github-action-ci/**'
pull_request:
branches:
- main
paths:
- .github/workflows/build-ci-container.yml
- '.github/workflows/containers/github-action-ci/**'

jobs:
build-ci-container:
if: github.repository_owner == 'llvm'
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- name: Write Variables
id: vars
run: |
tag=`date +%s`
container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/ci-ubuntu-22.04"
echo "container-name=$container_name" >> $GITHUB_OUTPUT
echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
- name: Checkout LLVM
uses: actions/checkout@v4
with:
sparse-checkout: .github/workflows/containers/github-action-ci/

- name: Build Container
working-directory: ./.github/workflows/containers/github-action-ci/
run: |
podman build -t ${{ steps.vars.outputs.container-name-tag }} .
podman tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest
- name: Test Container
run: |
for image in ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}; do
podman run --rm -it $image /usr/bin/bash -x -c 'printf '\''#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello'
done
- name: Push Container
if: github.event_name == 'push'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
podman push ${{ steps.vars.outputs.container-name-tag }}
podman push ${{ steps.vars.outputs.container-name }}:latest
48 changes: 48 additions & 0 deletions .github/workflows/containers/github-action-ci/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
FROM docker.io/library/ubuntu:22.04 as base
ENV LLVM_SYSROOT=/opt/llvm/

FROM base as toolchain
ENV LLVM_MAJOR=17
ENV LLVM_VERSION=${LLVM_MAJOR}.0.6
ENV LLVM_DIRNAME=clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-22.04
ENV LLVM_FILENAME=${LLVM_DIRNAME}.tar.xz

RUN apt-get update && \
apt-get install -y \
curl \
xz-utils

RUN mkdir -p $LLVM_SYSROOT/bin/ $LLVM_SYSROOT/lib/

RUN curl -O -L https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/$LLVM_FILENAME

RUN tar -C $LLVM_SYSROOT --strip-components=1 -xJf $LLVM_FILENAME \
$LLVM_DIRNAME/bin/clang \
$LLVM_DIRNAME/bin/clang++ \
$LLVM_DIRNAME/bin/clang-cl \
$LLVM_DIRNAME/bin/clang-$LLVM_MAJOR \
$LLVM_DIRNAME/bin/lld \
$LLVM_DIRNAME/bin/ld.lld \
$LLVM_DIRNAME/lib/clang/


FROM base

COPY --from=toolchain $LLVM_SYSROOT $LLVM_SYSROOT

# Need to install curl for hendrikmuhs/ccache-action
# Need nodejs for some of the GitHub actions.
# Need perl-modules for clang analyzer tests.
RUN apt-get update && \
apt-get install -y \
binutils \
cmake \
curl \
libstdc++-11-dev \
ninja-build \
nodejs \
perl-modules \
python3-psutil

ENV LLVM_SYSROOT=$LLVM_SYSROOT
ENV PATH=${LLVM_SYSROOT}/bin:${PATH}
1 change: 0 additions & 1 deletion .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -167,4 +167,3 @@ jobs:
run: |
cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON -DSPHINX_WARNINGS_AS_ERRORS=OFF ./llvm
TZ=UTC ninja -C flang-build docs-flang-html docs-flang-man
39 changes: 39 additions & 0 deletions .github/workflows/libclang-python-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
name: Libclang Python Binding Tests

permissions:
contents: read

on:
workflow_dispatch:
push:
paths:
- 'clang/bindings/python/**'
- 'clang/tools/libclang/**'
- 'clang/CMakeList.txt'
- '.github/workflows/libclang-python-tests.yml'
- '.github/workflows/llvm-project-tests.yml'
pull_request:
paths:
- 'clang/bindings/python/**'
- 'clang/tools/libclang/**'
- 'clang/CMakeList.txt'
- '.github/workflows/libclang-python-tests.yml'
- '.github/workflows/llvm-project-tests.yml'

concurrency:
# Skip intermediate builds: always.
# Cancel intermediate builds: only if it is a pull request build.
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}

jobs:
check-clang-python:
# Build libclang and then run the libclang Python binding's unit tests.
name: Build and run Python unit tests
uses: ./.github/workflows/llvm-project-tests.yml
with:
build_target: check-clang-python
projects: clang
# There is an issue running on "windows-2019".
# See https://github.com/llvm/llvm-project/issues/76601#issuecomment-1873049082.
os_list: '["ubuntu-latest"]'
3 changes: 1 addition & 2 deletions .github/workflows/libcxx-build-and-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,8 @@ jobs:
'generic-no-unicode',
'generic-no-wide-characters',
'generic-no-rtti',
'generic-optimized-speed',
'generic-static',
'generic-with_llvm_unwinder',
# TODO Find a better place for the benchmark and bootstrapping builds to live. They're either very expensive
# or don't provide much value since the benchmark run results are too noise on the bots.
'benchmarks',
Expand Down Expand Up @@ -207,4 +207,3 @@ jobs:
**/CMakeError.log
**/CMakeOutput.log
**/crash_diagnostics/*
2 changes: 1 addition & 1 deletion .github/workflows/llvm-project-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ jobs:
# enough cache space for all the tests to run at once and still
# fit under the 10 GB limit.
max-size: 500M
key: sccache-${{ matrix.os }}
key: ${{ matrix.os }}
variant: sccache
- name: Build and Test
uses: llvm/actions/build-test-llvm-project@main
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/release-binaries.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ jobs:

- name: Build Clang
run: |
cmake -G Ninja -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_BUILD_TYPE=Release -DCMAKE_ENABLE_ASSERTIONS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DLLVM_ENABLE_PROJECTS=clang -S llvm -B build
ninja -v -C build
cmake -G Ninja -C clang/cmake/caches/Release.cmake -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_POSITION_INDEPENDENT_CODE=ON -S llvm -B build
ninja -v -C build clang
build-binaries:
Expand Down Expand Up @@ -152,6 +152,7 @@ jobs:
-triple ${{ matrix.target.triple }} \
-use-ninja \
-no-checkout \
-use-cmake-cache \
-no-test-suite \
-configure-flags "-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# The LLVM Compiler Infrastructure

[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/llvm/llvm-project/badge)](https://securityscorecards.dev/viewer/?uri=github.com/llvm/llvm-project)
[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8273/badge)](https://www.bestpractices.dev/projects/8273)
[![libc++](https://github.com/llvm/llvm-project/actions/workflows/libcxx-build-and-test.yaml/badge.svg?branch=main&event=schedule)](https://github.com/llvm/llvm-project/actions/workflows/libcxx-build-and-test.yaml?query=event%3Aschedule)

Welcome to the LLVM project!

Expand Down
93 changes: 93 additions & 0 deletions bolt/docs/BAT.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# BOLT Address Translation (BAT)
# Purpose
A regular profile collection for BOLT involves collecting samples from
unoptimized binary. BOLT Address Translation allows collecting profile
from BOLT-optimized binary and using it for optimizing the input (pre-BOLT)
binary.

# Overview
BOLT Address Translation is an extra section (`.note.bolt_bat`) inserted by BOLT
into the output binary containing translation tables and split functions linkage
information. This information enables mapping the profile back from optimized
binary onto the original binary.

# Usage
`--enable-bat` flag controls the generation of BAT section. Sampled profile
needs to be passed along with the optimized binary containing BAT section to
`perf2bolt` which reads BAT section and produces fdata profile for the original
binary. Note that YAML profile generation is not supported since BAT doesn't
contain the metadata for input functions.

# Internals
## Section contents
The section is organized as follows:
- Hot functions table
- Address translation tables
- Cold functions table

## Construction and parsing
BAT section is created from `BoltAddressTranslation` class which captures
address translation information provided by BOLT linker. It is then encoded as a
note section in the output binary.

During profile conversion when BAT-enabled binary is passed to perf2bolt,
`BoltAddressTranslation` class is populated from BAT section. The class is then
queried by `DataAggregator` during sample processing to reconstruct addresses/
offsets in the input binary.

## Encoding format
The encoding is specified in
[BoltAddressTranslation.h](/bolt/include/bolt/Profile/BoltAddressTranslation.h)
and [BoltAddressTranslation.cpp](/bolt/lib/Profile/BoltAddressTranslation.cpp).

### Layout
The general layout is as follows:
```
Hot functions table header
|------------------|
| Function entry |
| |--------------| |
| | OutOff InOff | |
| |--------------| |
~~~~~~~~~~~~~~~~~~~~
Cold functions table header
|------------------|
| Function entry |
| |--------------| |
| | OutOff InOff | |
| |--------------| |
~~~~~~~~~~~~~~~~~~~~
```

### Functions table
Hot and cold functions tables share the encoding except difference marked below.
Header:
| Entry | Encoding | Description |
| ------ | ----- | ----------- |
| `NumFuncs` | ULEB128 | Number of functions in the functions table |

The header is followed by Functions table with `NumFuncs` entries.
Output binary addresses are delta encoded, meaning that only the difference with
the previous output address is stored. Addresses implicitly start at zero.
Hot indices are delta encoded, implicitly starting at zero.
| Entry | Encoding | Description |
| ------ | ------| ----------- |
| `Address` | Delta, ULEB128 | Function address in the output binary |
| `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table |
| `NumEntries` | ULEB128 | Number of address translation entries for a function |

Function header is followed by `NumEntries` pairs of offsets for current
function.

### Address translation table
Delta encoding means that only the difference with the previous corresponding
entry is encoded. Offsets implicitly start at zero.
| Entry | Encoding | Description |
| ------ | ------| ----------- |
| `OutputOffset` | Delta, ULEB128 | Function offset in output binary |
| `InputOffset` | Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit |

`BRANCHENTRY` bit denotes whether a given offset pair is a control flow source
(branch or call instruction). If not set, it signifies a control flow target
(basic block offset).
13 changes: 12 additions & 1 deletion bolt/include/bolt/Profile/BoltAddressTranslation.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/DataExtractor.h"
#include <cstdint>
#include <map>
#include <optional>
Expand Down Expand Up @@ -118,14 +119,24 @@ class BoltAddressTranslation {
void writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB,
uint64_t FuncAddress);

/// Write the serialized address translation table for a function.
template <bool Cold>
void writeMaps(std::map<uint64_t, MapTy> &Maps, raw_ostream &OS);

/// Read the serialized address translation table for a function.
/// Return a parse error if failed.
template <bool Cold>
void parseMaps(std::vector<uint64_t> &HotFuncs, DataExtractor &DE,
uint64_t &Offset, Error &Err);

std::map<uint64_t, MapTy> Maps;

/// Links outlined cold bocks to their original function
std::map<uint64_t, uint64_t> ColdPartSource;

/// Identifies the address of a control-flow changing instructions in a
/// translation map entry
const static uint32_t BRANCHENTRY = 0x80000000;
const static uint32_t BRANCHENTRY = 0x1;
};
} // namespace bolt

Expand Down
Loading

0 comments on commit 18d06c2

Please sign in to comment.