Merge branch 'main' into format3

.ci/monolithic-linux.sh

@@ -45,7 +45,7 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
       -D COMPILER_RT_BUILD_LIBFUZZER=OFF \
-      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \
+      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
       -D LLVM_ENABLE_LLD=ON \
       -D CMAKE_CXX_FLAGS=-gmlt \
       -D BOLT_CLANG_EXE=/usr/bin/clang \

.ci/monolithic-windows.sh

@@ -45,7 +45,7 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
       -D COMPILER_RT_BUILD_LIBFUZZER=OFF \
-      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \
+      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \
       -D COMPILER_RT_BUILD_ORC=OFF \
       -D CMAKE_C_COMPILER_LAUNCHER=sccache \
       -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \

.github/workflows/README.md

@@ -1 +1 @@
-Github action workflows should be stored in this directrory.
+Github action workflows should be stored in this directory.

.github/workflows/build-ci-container.yml

@@ -0,0 +1,60 @@
+
+name: Build CI Container
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/build-ci-container.yml
+      - '.github/workflows/containers/github-action-ci/**'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .github/workflows/build-ci-container.yml
+      - '.github/workflows/containers/github-action-ci/**'
+
+jobs:
+  build-ci-container:
+    if: github.repository_owner == 'llvm'
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Write Variables
+        id: vars
+        run: |
+          tag=`date +%s`
+          container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/ci-ubuntu-22.04"
+          echo "container-name=$container_name" >> $GITHUB_OUTPUT
+          echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
+
+      - name: Checkout LLVM
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/workflows/containers/github-action-ci/
+
+      - name: Build Container
+        working-directory: ./.github/workflows/containers/github-action-ci/
+        run: |
+          podman build -t ${{ steps.vars.outputs.container-name-tag }} .
+          podman tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest
+
+      - name: Test Container
+        run: |
+          for image in ${{ steps.vars.outputs.container-name-tag }} ${{  steps.vars.outputs.container-name }}; do
+            podman run --rm -it $image /usr/bin/bash -x -c 'printf '\''#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello'
+          done
+
+      - name: Push Container
+        if: github.event_name == 'push'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
+          podman push ${{ steps.vars.outputs.container-name-tag }}
+          podman push ${{ steps.vars.outputs.container-name }}:latest

.github/workflows/containers/github-action-ci/Dockerfile

@@ -0,0 +1,48 @@
+FROM docker.io/library/ubuntu:22.04 as base
+ENV LLVM_SYSROOT=/opt/llvm/
+
+FROM base as toolchain
+ENV LLVM_MAJOR=17
+ENV LLVM_VERSION=${LLVM_MAJOR}.0.6
+ENV LLVM_DIRNAME=clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-22.04
+ENV LLVM_FILENAME=${LLVM_DIRNAME}.tar.xz
+
+RUN apt-get update && \
+    apt-get install -y \
+    curl \
+    xz-utils
+
+RUN mkdir -p $LLVM_SYSROOT/bin/ $LLVM_SYSROOT/lib/
+
+RUN curl -O -L https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/$LLVM_FILENAME
+
+RUN tar -C $LLVM_SYSROOT --strip-components=1 -xJf $LLVM_FILENAME \
+    $LLVM_DIRNAME/bin/clang \
+    $LLVM_DIRNAME/bin/clang++ \
+    $LLVM_DIRNAME/bin/clang-cl \
+    $LLVM_DIRNAME/bin/clang-$LLVM_MAJOR \
+    $LLVM_DIRNAME/bin/lld \
+    $LLVM_DIRNAME/bin/ld.lld \
+    $LLVM_DIRNAME/lib/clang/
+
+
+FROM base
+
+COPY --from=toolchain $LLVM_SYSROOT $LLVM_SYSROOT
+
+# Need to install curl for hendrikmuhs/ccache-action
+# Need nodejs for some of the GitHub actions.
+# Need perl-modules for clang analyzer tests.
+RUN apt-get update && \
+    apt-get install -y \
+    binutils \
+    cmake \
+    curl \
+    libstdc++-11-dev \
+    ninja-build \
+    nodejs \
+    perl-modules \
+    python3-psutil
+
+ENV LLVM_SYSROOT=$LLVM_SYSROOT
+ENV PATH=${LLVM_SYSROOT}/bin:${PATH}

.github/workflows/docs.yml

@@ -167,4 +167,3 @@ jobs:
         run: |
           cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON -DSPHINX_WARNINGS_AS_ERRORS=OFF ./llvm
           TZ=UTC ninja -C flang-build docs-flang-html docs-flang-man
-

.github/workflows/libclang-python-tests.yml

@@ -0,0 +1,39 @@
+name: Libclang Python Binding Tests
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'clang/bindings/python/**'
+      - 'clang/tools/libclang/**'
+      - 'clang/CMakeList.txt'
+      - '.github/workflows/libclang-python-tests.yml'
+      - '.github/workflows/llvm-project-tests.yml'
+  pull_request:
+    paths:
+      - 'clang/bindings/python/**'
+      - 'clang/tools/libclang/**'
+      - 'clang/CMakeList.txt'
+      - '.github/workflows/libclang-python-tests.yml'
+      - '.github/workflows/llvm-project-tests.yml'
+
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  check-clang-python:
+    # Build libclang and then run the libclang Python binding's unit tests.
+    name: Build and run Python unit tests
+    uses: ./.github/workflows/llvm-project-tests.yml
+    with:
+      build_target: check-clang-python
+      projects: clang
+      # There is an issue running on "windows-2019".
+      # See https://github.com/llvm/llvm-project/issues/76601#issuecomment-1873049082.
+      os_list: '["ubuntu-latest"]'

.github/workflows/libcxx-build-and-test.yaml

@@ -161,8 +161,8 @@ jobs:
           'generic-no-unicode',
           'generic-no-wide-characters',
           'generic-no-rtti',
+          'generic-optimized-speed',
           'generic-static',
-          'generic-with_llvm_unwinder',
           # TODO Find a better place for the benchmark and bootstrapping builds to live. They're either very expensive
           # or don't provide much value since the benchmark run results are too noise on the bots.
           'benchmarks',
@@ -207,4 +207,3 @@ jobs:
             **/CMakeError.log
             **/CMakeOutput.log
             **/crash_diagnostics/*
-  

.github/workflows/llvm-project-tests.yml

@@ -87,7 +87,7 @@ jobs:
           # enough cache space for all the tests to run at once and still
           # fit under the 10 GB limit.
           max-size: 500M
-          key: sccache-${{ matrix.os }}
+          key: ${{ matrix.os }}
           variant: sccache
       - name: Build and Test
         uses: llvm/actions/build-test-llvm-project@main

.github/workflows/release-binaries.yml

@@ -93,8 +93,8 @@ jobs:
 
     - name: Build Clang
       run: |
-        cmake -G Ninja  -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_BUILD_TYPE=Release -DCMAKE_ENABLE_ASSERTIONS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DLLVM_ENABLE_PROJECTS=clang -S llvm -B build
-        ninja -v -C build
+        cmake -G Ninja -C clang/cmake/caches/Release.cmake -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_POSITION_INDEPENDENT_CODE=ON -S llvm -B build
+        ninja -v -C build clang
 
 
   build-binaries:
@@ -152,6 +152,7 @@ jobs:
         -triple ${{ matrix.target.triple }} \
         -use-ninja \
         -no-checkout \
+        -use-cmake-cache \
         -no-test-suite \
         -configure-flags "-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
 

README.md

@@ -1,6 +1,8 @@
 # The LLVM Compiler Infrastructure
 
 [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/llvm/llvm-project/badge)](https://securityscorecards.dev/viewer/?uri=github.com/llvm/llvm-project)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8273/badge)](https://www.bestpractices.dev/projects/8273)
+[![libc++](https://github.com/llvm/llvm-project/actions/workflows/libcxx-build-and-test.yaml/badge.svg?branch=main&event=schedule)](https://github.com/llvm/llvm-project/actions/workflows/libcxx-build-and-test.yaml?query=event%3Aschedule)
 
 Welcome to the LLVM project!
 

bolt/docs/BAT.md

@@ -0,0 +1,93 @@
+# BOLT Address Translation (BAT)
+# Purpose
+A regular profile collection for BOLT involves collecting samples from
+unoptimized binary. BOLT Address Translation allows collecting profile
+from BOLT-optimized binary and using it for optimizing the input (pre-BOLT)
+binary.
+
+# Overview
+BOLT Address Translation is an extra section (`.note.bolt_bat`) inserted by BOLT
+into the output binary containing translation tables and split functions linkage
+information. This information enables mapping the profile back from optimized
+binary onto the original binary.
+
+# Usage
+`--enable-bat` flag controls the generation of BAT section. Sampled profile
+needs to be passed along with the optimized binary containing BAT section to
+`perf2bolt` which reads BAT section and produces fdata profile for the original
+binary. Note that YAML profile generation is not supported since BAT doesn't
+contain the metadata for input functions.
+
+# Internals
+## Section contents
+The section is organized as follows:
+- Hot functions table
+  - Address translation tables
+- Cold functions table
+
+## Construction and parsing
+BAT section is created from `BoltAddressTranslation` class which captures
+address translation information provided by BOLT linker. It is then encoded as a
+note section in the output binary.
+
+During profile conversion when BAT-enabled binary is passed to perf2bolt,
+`BoltAddressTranslation` class is populated from BAT section. The class is then
+queried by `DataAggregator` during sample processing to reconstruct addresses/
+offsets in the input binary.
+
+## Encoding format
+The encoding is specified in
+[BoltAddressTranslation.h](/bolt/include/bolt/Profile/BoltAddressTranslation.h)
+and [BoltAddressTranslation.cpp](/bolt/lib/Profile/BoltAddressTranslation.cpp).
+
+### Layout
+The general layout is as follows:
+```
+Hot functions table header
+|------------------|
+|  Function entry  |
+| |--------------| |
+| | OutOff InOff | |
+| |--------------| |
+~~~~~~~~~~~~~~~~~~~~
+
+Cold functions table header
+|------------------|
+|  Function entry  |
+| |--------------| |
+| | OutOff InOff | |
+| |--------------| |
+~~~~~~~~~~~~~~~~~~~~
+```
+
+### Functions table
+Hot and cold functions tables share the encoding except difference marked below.
+Header:
+| Entry  | Encoding | Description |
+| ------ | ----- | ----------- |
+| `NumFuncs` | ULEB128 | Number of functions in the functions table |
+
+The header is followed by Functions table with `NumFuncs` entries.
+Output binary addresses are delta encoded, meaning that only the difference with
+the previous output address is stored. Addresses implicitly start at zero.
+Hot indices are delta encoded, implicitly starting at zero.
+| Entry  | Encoding | Description |
+| ------ | ------| ----------- |
+| `Address` | Delta, ULEB128 | Function address in the output binary |
+| `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table |
+| `NumEntries` | ULEB128 | Number of address translation entries for a function |
+
+Function header is followed by `NumEntries` pairs of offsets for current
+function.
+
+### Address translation table
+Delta encoding means that only the difference with the previous corresponding
+entry is encoded. Offsets implicitly start at zero.
+| Entry  | Encoding | Description |
+| ------ | ------| ----------- |
+| `OutputOffset` | Delta, ULEB128 | Function offset in output binary |
+| `InputOffset` | Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit |
+
+`BRANCHENTRY` bit denotes whether a given offset pair is a control flow source
+(branch or call instruction). If not set, it signifies a control flow target
+(basic block offset).

bolt/include/bolt/Profile/BoltAddressTranslation.h

@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/DataExtractor.h"
 #include <cstdint>
 #include <map>
 #include <optional>
@@ -118,14 +119,24 @@ class BoltAddressTranslation {
   void writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB,
                          uint64_t FuncAddress);
 
+  /// Write the serialized address translation table for a function.
+  template <bool Cold>
+  void writeMaps(std::map<uint64_t, MapTy> &Maps, raw_ostream &OS);
+
+  /// Read the serialized address translation table for a function.
+  /// Return a parse error if failed.
+  template <bool Cold>
+  void parseMaps(std::vector<uint64_t> &HotFuncs, DataExtractor &DE,
+                 uint64_t &Offset, Error &Err);
+
   std::map<uint64_t, MapTy> Maps;
 
   /// Links outlined cold bocks to their original function
   std::map<uint64_t, uint64_t> ColdPartSource;
 
   /// Identifies the address of a control-flow changing instructions in a
   /// translation map entry
-  const static uint32_t BRANCHENTRY = 0x80000000;
+  const static uint32_t BRANCHENTRY = 0x1;
 };
 } // namespace bolt