diff --git a/.github/workflows/build-scheds.yml b/.github/workflows/build-scheds.yml deleted file mode 100644 index 5dbdc9ea9..000000000 --- a/.github/workflows/build-scheds.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: build-scheds -run-name: ${{ github.actor }} PR run -on: [pull_request] -jobs: - lint: - runs-on: ubuntu-22.04 - steps: - - name: Install deps - run: | - curl https://sh.rustup.rs -sSf | RUSTUP_INIT_SKIP_PATH_CHECK=yes sh -s -- -y - rustup install nightly-2024-09-10 - export PATH="~/.cargo/bin:$PATH" - - - uses: actions/checkout@v4 - - # Lint code - - run: cargo +nightly-2024-09-10 fmt - - run: git diff --exit-code - - build-schedulers: - runs-on: ubuntu-22.04 - steps: - ### OTHER REPOS #### - - # Hard turn-off interactive mode - - run: echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections - - # Refresh packages list - - run: sudo apt update - - ### DOWNLOAD AND INSTALL DEPENDENCIES ### - - # Download dependencies packaged by Ubuntu - - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd - - # clang 17 - # Use a custom llvm.sh script which includes the -y flag for - # add-apt-repository. Otherwise, the CI job will hang. If and when - # https://github.com/opencollab/llvm-jenkins.debian.net/pull/26 is - # merged, we can go back to using https://apt.llvm.org/llvm.sh. - - run: wget https://raw.githubusercontent.com/Decave/llvm-jenkins.debian.net/fix_llvmsh/llvm.sh - - run: chmod +x llvm.sh - - run: sudo ./llvm.sh all - - run: sudo ln -sf /usr/bin/clang-17 /usr/bin/clang - - run: sudo ln -sf /usr/bin/llvm-strip-17 /usr/bin/llvm-strip - - - uses: actions/checkout@v4 - - # meson - - run: pip install meson - - # Install virtme-ng - - run: pip install virtme-ng - - # Get the latest sched-ext enabled kernel directly from the korg - # for-next branch - - run: git clone --single-branch -b for-next --depth 1 https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git linux - - # Print the latest commit of the checked out sched-ext kernel - - run: cd linux && git log -1 --pretty=format:"%h %ad %s" --date=short - - # Build a minimal kernel (with sched-ext enabled) using virtme-ng - - run: cd linux && vng -v --build --config ../.github/workflows/sched-ext.config - - # Generate kernel headers - - run: cd linux && make headers - - # veristat - - run: wget https://github.com/libbpf/veristat/releases/download/v0.3.2/veristat-v0.3.2-amd64.tar.gz - - run: tar -xvf veristat-v0.3.2-amd64.tar.gz && sudo cp veristat /usr/bin/ - - run: sudo chmod +x /usr/bin/veristat && sudo chmod 755 /usr/bin/veristat - - ### END DEPENDENCIES ### - - # The actual build: - - run: meson setup build -Dkernel=$(pwd)/linux -Dkernel_headers=./linux/usr/include -Denable_stress=true - - run: meson compile -C build - - # Print CPU model before running the tests (this can be useful for - # debugging purposes) - - run: grep 'model name' /proc/cpuinfo | head -1 - - # Setup KVM support - - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - # Test schedulers - - run: meson compile -C build test_sched - # Stress schedulers - - run: meson compile -C build stress_tests - - run: meson compile -C build veristat diff --git a/.github/workflows/caching-build.yml b/.github/workflows/caching-build.yml index 3d1f89a03..5be76ffb7 100644 --- a/.github/workflows/caching-build.yml +++ b/.github/workflows/caching-build.yml @@ -221,43 +221,3 @@ jobs: - run: cargo build --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml - run: cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml --no-run - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml - - pages: - runs-on: ubuntu-24.04 - if: github.ref == 'refs/heads/main' - permissions: - contents: write # To push a branch - pages: write # To push to a GitHub Pages site - id-token: write # To update the deployment status - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: ./.github/actions/install-deps-action - - name: Build Book and Linux Docs - run: | - curl https://sh.rustup.rs -sSf | RUSTUP_INIT_SKIP_PATH_CHECK=yes sh -s -- -y - rustup install nightly - export PATH="~/.cargo/bin:$PATH" - RUSTDOCFLAGS="--enable-index-page -Zunstable-options" ~/.cargo/bin/cargo +nightly doc --workspace --no-deps --bins --lib --examples --document-private-items --all-features - sudo apt update - sudo apt install build-essential graphviz sphinx-doc python3-sphinx-rtd-theme texlive-latex-recommended python3-yaml -y - cargo install htmlq - git clone --single-branch -b for-next --depth 1 https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git linux - cd linux - make DOCS_THEME=sphinx_rtd_theme SPHINXDIRS=scheduler htmldocs - make DOCS_THEME=sphinx_rtd_theme SPHINXDIRS=bpf htmldocs - cd .. - cp -a linux/Documentation/output/scheduler target/doc/ - cp -a linux/Documentation/output/bpf target/doc/ - sed -i 's%
  • server
  • %
  • server
  • Kernel Sched Ext Docs
  • Kernel Bpf Helpers Docs
  • %' target/doc/index.html - - name: Setup Pages - uses: actions/configure-pages@v4 - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: 'target/doc' - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 - diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml deleted file mode 100644 index 1f894e635..000000000 --- a/.github/workflows/merge.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: merge -on: - pull_request_target: - types: - - closed - -jobs: - if_merged: - if: github.event.pull_request.merged == true - runs-on: ubuntu-22.04 - steps: - - name: Restore veristat values - id: cache-veristat-restore - uses: actions/cache/restore@v4 - with: - path: ${{ github.workspace }}/veristat - key: veristat-diffs-main - - # Hard turn-off interactive mode - - run: echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections - - # Refresh packages list - - run: sudo apt update - - ### DOWNLOAD AND INSTALL DEPENDENCIES ### - - # Download dependencies packaged by Ubuntu - - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd - - # clang 17 - # Use a custom llvm.sh script which includes the -y flag for - # add-apt-repository. Otherwise, the CI job will hang. If and when - # https://github.com/opencollab/llvm-jenkins.debian.net/pull/26 is - # merged, we can go back to using https://apt.llvm.org/llvm.sh. - - run: wget https://raw.githubusercontent.com/Decave/llvm-jenkins.debian.net/fix_llvmsh/llvm.sh - - run: chmod +x llvm.sh - - run: sudo ./llvm.sh all - - run: sudo ln -sf /usr/bin/clang-17 /usr/bin/clang - - run: sudo ln -sf /usr/bin/llvm-strip-17 /usr/bin/llvm-strip - - - uses: actions/checkout@v4 - - # meson - - run: pip install meson - - # Install virtme-ng - - run: pip install virtme-ng - - # Get the latest sched-ext enabled kernel directly from the korg - # for-next branch - - run: git clone --single-branch -b for-next --depth 1 https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git linux - - # Print the latest commit of the checked out sched-ext kernel - - run: cd linux && git log -1 --pretty=format:"%h %ad %s" --date=short - - # Build a minimal kernel (with sched-ext enabled) using virtme-ng - - run: cd linux && vng -v --build --config ../.github/workflows/sched-ext.config - - # Generate kernel headers - - run: cd linux && make headers - - - run: wget https://github.com/libbpf/veristat/releases/download/v0.3.2/veristat-v0.3.2-amd64.tar.gz - - run: tar -xvf veristat-v0.3.2-amd64.tar.gz && sudo cp veristat /usr/bin/ - - run: sudo chmod +x /usr/bin/veristat && sudo chmod 755 /usr/bin/veristat - - ### END DEPENDENCIES ### - - # The actual build: - - run: meson setup build -Dkernel=$(pwd)/linux -Dkernel_headers=./linux/usr/include -Dveristat_diff_dir=$GITHUB_WORKSPACE/veristat - - run: meson compile -C build - - # Print CPU model before running the tests (this can be useful for - # debugging purposes) - - run: grep 'model name' /proc/cpuinfo | head -1 - - # Setup KVM support - - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - # run veristat - - run: meson compile -C build veristat_diff - - # update veristat cache dirs - - run: cd $GITHUB_WORKSPACE/veristat && for file in *_new.csv; do mv "$file" "${file%_new.csv}.csv"; done || echo "failed to update veristat cache" - - - name: Save veristat cache - id: cache-veristat-save - uses: actions/cache/save@v4 - with: - path: ${{ github.workspace }}/veristat - key: veristat-diffs-main diff --git a/README.md b/README.md index 553105993..e06cf7a83 100644 --- a/README.md +++ b/README.md @@ -1,481 +1,40 @@ -# Sched_ext Schedulers and Tools +# Backports of scx to older kernels -[sched_ext](https://github.com/sched-ext/scx) is a Linux kernel feature -which enables implementing kernel thread schedulers in BPF and dynamically -loading them. This repository contains various scheduler implementations and -support utilities. +***Work In Progress*** -sched_ext enables safe and rapid iterations of scheduler implementations, thus -radically widening the scope of scheduling strategies that can be experimented -with and deployed; even in massive and complex production environments. +The goal of this repo is to enable sched-ext binaries to build +on kernels with older versions of the sched-ext source tree. -- The [scx_layered case - study](https://github.com/sched-ext/scx/blob/case-studies/case-studies/scx_layered.md) - concretely demonstrates the power and benefits of sched_ext. -- For a high-level but thorough overview of the sched_ext (especially its - motivation), please refer to the [overview document](OVERVIEW.md). -- For a description of the schedulers shipped with this tree, please refer to - the [schedulers document](scheds/README.md). -- The following video is the [scx_rustland](https://github.com/sched-ext/scx/tree/main/scheds/rust/scx_rustland) - scheduler which makes most scheduling decisions in userspace Rust code showing - better FPS in terraria while kernel is being compiled. This doesn't mean that - scx_rustland is a better scheduler but does demonstrate how safe and easy it is to - implement a scheduler which is generally usable and can outperform the default - scheduler in certain scenarios. +## Dependencies +* Docker -[scx_rustland-terraria](https://github.com/sched-ext/scx/assets/1051723/42ec3bf2-9f1f-4403-80ab-bf5d66b7c2d5) +## How to use this repo -While the kernel feature is not upstream yet, we believe sched_ext has a -reasonable chance of landing upstream in the foreseeable future. Both Meta -and Google are fully committed to sched_ext and Meta is in the process of -mass production deployment. See (#kernel-feature-status) for more details. +### Setting up a backport -In all example shell commands, `$SCX` refers to the root of this repository. +1) Create a branch named `$SHORT_SHA-vmlinux` , where `$SHORT_SHA` is the first 7 characters of the commit hash in the scx kernel tree you are targeting. +2) Run `backport-scripts/generate-backport-vmlinux.sh` passing it `$SHORT_SHA` (such as `af1234`) and branch name, in that order. Optionally, pass a third +arg, the git repo to obtain that commit and branch from, if neccessary. -## Getting Started +3) Update the symlink `scheds/include/vmlinux/vmlinux.h` to point to your new vmlinux.h. -All that's necessary for running sched_ext schedulers is a kernel with -sched_ext support and the scheduler binaries along with the libraries they -depend on. Switching to a sched_ext scheduler is as simple as running a -sched_ext binary: +4) Add a CI badge to this branch on this page. -``` -root@test ~# cat /sys/kernel/sched_ext/state /sys/kernel/sched_ext/*/ops 2>/dev/null -disabled -root@test ~# scx_simple -local=1 global=0 -local=74 global=15 -local=78 global=32 -local=82 global=42 -local=86 global=54 -^Zfish: Job 1, 'scx_simple' has stopped -root@test ~# cat /sys/kernel/sched_ext/state /sys/kernel/sched_ext/*/ops 2>/dev/null -enabled -simple -root@test ~# fg -Send job 1 (scx_simple) to foreground -local=635 global=179 -local=696 global=192 -^CEXIT: BPF scheduler unregistered -``` +5) Commit and push your branch. -[`scx_simple`](https://github.com/sched-ext/scx/blob/main/scheds/c/scx_simple.bpf.c) -is a very simple global vtime scheduler which can behave acceptably on CPUs -with a simple topology (single socket and single L3 cache domain). +6) Open a new branch (i.e. `af1234-${SCX_RELEASE}-fixes`) off of your initial branch (i.e. `af1234`) +and edit files other than `vmlinux.h` until first `cargo test` (ran in this repo's root) passes and then CI passes. -Above, we switch the whole system to use `scx_simple` by running the binary, -suspend it with `ctrl-z` to confirm that it's loaded, and then switch back -to the kernel default scheduler by terminating the process with `ctrl-c`. -For `scx_simple`, suspending the scheduler process doesn't affect scheduling -behavior because all that the userspace component does is print statistics. -This doesn't hold for all schedulers. +7) Push this branch as you get things working, push it as `af1234-${SCX_RELEASE}-backport` once CI passes. -In addition to terminating the program, there are two more ways to disable a -sched_ext scheduler - `sysrq-S` and the watchdog timer. Ignoring kernel -bugs, the worst damage a sched_ext scheduler can do to a system is starving -some threads until the watchdog timer triggers. +### Updating a backport -As illustrated, once the kernel and binaries are in place, using sched_ext -schedulers is straightforward and safe. While developing and building -schedulers in this repository isn't complicated either, sched_ext makes use -of many new BPF features, some of which require build tools which are newer -than what many distros are currently shipping. This should become less of an -issue in the future. For the time being, the following custom repositories -are provided for select distros. +1) Create a new `-fixes` branch from either a `-vmlinux`, `-fixes` branch or a `-backport` branch (whichever is closest to what you want). +2) Continue the instructions above from (4). -## Install Instructions by Distro +### Updating this doc and the setup scripts. -- [Ubuntu](INSTALL.md#ubuntu) -- [Arch Linux](INSTALL.md#arch-linux) -- [Fedora](INSTALL.md#fedora) -- [Nix](INSTALL.md#nix) +Please make any updates to this document and the setup scripts on the branch `setup-repo`. This will be rebased atop `main` when merging in updates. -## Repository Structure - -``` -scx -|-- scheds : Sched_ext scheduler implementations -| |-- include : Shared BPF and user C include files including vmlinux.h -| |-- c : Example schedulers - userspace code written C -| \-- rust : Example schedulers - userspace code written Rust -\-- rust : Rust support code - \-- scx_utils : Common utility library for rust schedulers -``` - - -## Build & Install - -`meson` is the main build system but each Rust sub-project is its own -self-contained cargo project and can be built and published separately. The -followings are the dependencies and version requirements. - -**Note**: Many distros only have earlier versions of `meson`, in that case just [clone the meson -repo](https://mesonbuild.com/Quick-guide.html#installation-from-source) and call -`meson.py` e.g. `/path/to/meson/repo/meson.py compile -C build`. Alternatively, use `pip` e.g. -`pip install meson` or `pip install meson --break-system-packages` (if needed). - -- `meson`: >=1.2, build scripts under `meson-scripts/` use `bash` and - standard utilities including `awk`. -- `clang`: >=16 required, >=17 recommended -- `libbpf`: >=1.2.2 required, >=1.3 recommended (`RESIZE_ARRAY` support is - new in 1.3). It's preferred to link statically against the source from the libbpf git repo, which is cloned during setup. -- Rust toolchain: >=1.72 -- `libelf`, `libz`, `libzstd` if linking against staic `libbpf.a` -- `bpftool` By default this is cloned and built as part of the default build process. Alternatively it's usually available in `linux-tools-common`. - -The kernel has to be built with the following configuration: -- `CONFIG_BPF=y` -- `CONFIG_BPF_EVENTS=y` -- `CONFIG_BPF_JIT=y` -- `CONFIG_BPF_SYSCALL=y` -- `CONFIG_DEBUG_INFO_BTF=y` -- `CONFIG_FTRACE=y` -- `CONFIG_SCHED_CLASS_EXT=y` - - -### Setting Up and Building - -`meson` always uses a separate build directory. Running the following -commands in the root of the tree builds and installs all schedulers under -`~/bin`. - -#### Static linking against libbpf (preferred) - - ``` -$ cd $SCX -$ meson setup build --prefix ~ -$ meson compile -C build -$ meson install -C build -``` - -Notes: `meson setup` will also clone both libbpf and bpftool repos and `meson compile` will build them both. - -Make sure you have dependencies installed that allow you to compile from source! - -##### Ubuntu/Debian - -``` -apt install gcc-multilib build-essential libssl-dev llvm lld libelf-dev -``` - -##### Arch Linux - -``` -pacman -S base-devel -``` - -### Static linking against system libbpf -Note, depending on your system configuration `libbpf_a` and `libbpf_h` may be -in different directories. The system libbpf version needs to match the minimum -libbpf version for scx. - ``` -$ cd $SCX -$ meson setup build --prefix ~ -D libbpf_a=/usr/lib64/libbpf.a libbpf_h=/usr/include/bpf/ -$ meson compile -C build -$ meson install -C build -``` - -#### Dynamic linking against libbpf -``` -$ cd $SCX -$ meson setup build --prefix ~ -D libbpf_a=disabled -$ meson compile -C build -$ meson install -C build -``` - -#### Using a different bpftool -This will check the system for an installed bpftool -``` -$ meson setup build --prefix ~ -D bpftool=disabled -``` -Using a custom built bpftool -``` -$ meson setup build --prefix ~ -D bpftool=/path/to/bpftool -``` - -Note that `meson compile` step is not strictly necessary as `install` -implies `compile`. The above also will build debug binaries with -optimizations turned off, which is useful for development but they aren't -optimized and big. For actual use you want to build release binaries. -`meson` uses `-D` argument to specify build options. The configuration -options can be specified at `setup` time but can also be changed afterwards -and `meson` will do the right thing. To switch to release builds, run the -following in the build directory and then compile and install again. - -``` -$ meson configure -Dbuildtype=release -``` - -Running `meson configure` without any argument shows all current build -options. For more information on `meson` arguments and built-in options, -please refer to `meson --help` and its -[documentation](https://mesonbuild.com/Builtin-options.html). - - -### Building Specific Schedulers and Binary Locations - -If you just want to build a subset of schedulers, you can specify the -scheduler names as arguments to `meson compile`. For example, if we just -want to build the simple example scheduler -`scheds/c/scx_simple` and the Rust userspace scheduler -`scheds/rust/scx_rusty`: - -``` -$ cd $SCX -$ meson setup build -Dbuildtype=release -$ meson compile -C build scx_simple scx_rusty -``` - -:warning: **If your system has `sccache` installed**: `meson` automatically -uses `sccache` if available. However, `sccache` fails in one of the build -steps. If you encounter this issue, disable `sccache` by specifying `CC` -directly - `$ CC=clang meson setup build -Dbuildtype=release`. - -You can also specify `-v` if you want to see the commands being used: - -``` -$ meson compile -C build -v scx_pair -``` - -For C userspace schedulers such as the ones under `scheds/c`, -the built binaries are located in the same directory under the build root. -For example, here, the `scx_simple` binary can be found at -`$SCX/build/scheds/c/scx_simple`. - -For Rust userspace schedulers such as the ones under `scheds/rust`, the -`scx_rusty` binary can be found at `$SCX/build/scheds/rust/release`. - - -### SCX specific build options - -While the default options should work in most cases, it may be desirable to -override some of the toolchains and dependencies - e.g. to directly use -`libbpf` built from the kernel source tree. The following `meson` build -options can be used in such cases. - -- `bpf_clang`: `clang` to use when compiling `.bpf.c` -- `bpftool`: `bpftool` to use when generating `.bpf.skel.h`. Set this to "disabled" to check the system for an already installed bpftool -- `libbpf_a`: Static `libbpf.a` to use. Set this to "disabled" to link libbpf dynamically -- `libbpf_h`: `libbpf` header directories, only meaningful with `libbpf_a` option -- `cargo`: `cargo` to use when building rust sub-projects -- 'cargo_home': 'CARGO_HOME env to use when invoking cargo' -- `offline`: 'Compilation step should not access the internet' -- `enable_rust`: 'Enable the build of rust sub-projects' - -For example, let's say you want to use `bpftool` and `libbpf` shipped in the -kernel tree located at `$KERNEL`. We need to build `bpftool` in the kernel -tree first, set up SCX build with the related options and then build & -install. - -``` -$ cd $KERNEL -$ make -C tools/bpf/bpftool -$ cd $SCX -$ BPFTOOL=$KERNEL/tools/bpf/bpftool -$ meson setup build -Dbuildtype=release -Dprefix=~/bin \ - -Dbpftool=$BPFTOOL/bpftool \ - -Dlibbpf_a=$BPFTOOL/libbpf/libbpf.a \ - -Dlibbpf_h=$BPFTOOL/libbpf/include -$ meson install -C build -``` - -Note that we use `libbpf` which was produced as a part of `bpftool` build -process rather than buliding `libbpf` directly. This is necessary because -`libbpf` header files need to be installed for them to be in the expected -relative locations. - - -### Offline Compilation - -Rust builds automatically download dependencies from crates.io; however, -some build environments might not allow internet access requiring all -dependencies to be available offline. The `fetch` target and `offline` -option are provided for such cases. - -The following downloads all Rust dependencies into `$HOME/cargo-deps`. - -``` -$ cd $SCX -$ meson setup build -Dcargo_home=$HOME/cargo-deps -$ meson compile -C build fetch -``` - -The following builds the schedulers without accessing the internet. The -`build` directory doesn't have to be the same one. The only requirement is -that the `cargo_home` option points to a directory which contains the -content generated from the previous step. - -``` -$ cd $SCX -$ meson setup build -Dcargo_home=$HOME/cargo-deps -Doffline=true -Dbuildtype=release -$ meson compile -C build -``` - -### Working with Rust Sub-projects - -Each Rust sub-project is its own self-contained cargo project. When buildng -as a part of this repository, `meson` invokes `cargo` with the appropriate -options and environment variables to sync the build environment. When -building separately by running `cargo build` directly in a sub-project -directory, it will automatically figure out build environment. Please take a -look at the -[`scx_utils::BpfBuilder`](https://docs.rs/scx_utils/latest/scx_utils/struct.BpfBuilder.html) -documentation for details. - -For example, the following builds and runs the `scx_rusty` scheduler: - -``` -$ cd $SCX/scheds/rust/scx_rusty -$ cargo build --release -$ cargo run --release -``` - -Here too, the `build` step is not strictly necessary as it's implied by -`run`. - -Note that Rust userspace schedulers are published on `crates.io` and can be -built and installed without cloning this repository as long as the necessary -toolchains are available. Simply run: - -``` -$ cargo install scx_rusty -``` - -and `scx_rusty` will be built and installed as `~/.cargo/bin/scx_rusty`. - -## Checking scx_stats - -With the implementation of scx_stats, schedulers no longer display statistics by default. To display the statistics from the currently running scheduler, a manual user action is required. -Below are examples of how to do this. - -- To check the scheduler statistics, use the - -``` -scx_SCHEDNAME --monitor $INTERVAL -``` - -for example 0.5 - this will print the output every half a second - -``` -scx_bpfland --monitor 0.5 -``` -Some schedulers may implement different or multiple monitoring options. Refer to --help of each scheduler for details. -Most schedulers also accept ` --stats $INTERVAL` to print the statistics directly from the scheduling instance. - -#### Examples - -- scx_bpfland - -``` -❯ scx_bpfland --monitor 5 -[scx_bpfland] tasks -> run: 3/4 int: 2 wait: 3 | nvcsw: 3 | dispatch -> dir: 0 prio: 73 shr: 9 -[scx_bpfland] tasks -> run: 4/4 int: 2 wait: 2 | nvcsw: 3 | dispatch -> dir: 1 prio: 3498 shr: 1385 -[scx_bpfland] tasks -> run: 4/4 int: 2 wait: 2 | nvcsw: 3 | dispatch -> dir: 1 prio: 2492 shr: 1311 -[scx_bpfland] tasks -> run: 4/4 int: 2 wait: 3 | nvcsw: 3 | dispatch -> dir: 2 prio: 3270 shr: 1748 -``` - -- scx_rusty - -``` -❯ scx_rusty --monitor 5 -###### Thu, 29 Aug 2024 14:42:37 +0200, load balance @ -265.1ms ###### -cpu= 0.00 load= 0.17 mig=0 task_err=0 lb_data_err=0 time_used= 0.0ms -tot= 15 sync_prev_idle= 0.00 wsync= 0.00 -prev_idle= 0.00 greedy_idle= 0.00 pin= 0.00 -dir= 0.00 dir_greedy= 0.00 dir_greedy_far= 0.00 -dsq=100.00 greedy_local= 0.00 greedy_xnuma= 0.00 -kick_greedy= 0.00 rep= 0.00 -dl_clamp=33.33 dl_preset=93.33 -slice=20000us -direct_greedy_cpus=f - kick_greedy_cpus=f - NODE[00] load= 0.17 imbal= +0.00 delta= +0.00 - DOM[00] load= 0.17 imbal= +0.00 delta= +0.00 -``` - -- scx_lavd - -``` -❯ scx_lavd --monitor 5 -| 12 | 1292 | 3 | 1 | 8510 | 37.6028 | 2.42068 | 99.1304 | 100 | 62.8907 | 100 | 100 | 62.8907 | performance | 100 | 0 | 0 | -| 13 | 2208 | 3 | 1 | 6142 | 33.3442 | 2.39336 | 98.7626 | 100 | 60.2084 | 100 | 100 | 60.2084 | performance | 100 | 0 | 0 | -| 14 | 941 | 3 | 1 | 5223 | 31.323 | 1.704 | 99.215 | 100.019 | 59.1614 | 100 | 100.019 | 59.1614 | performance | 100 | 0 | 0 | -``` - -- scx_rustland - -``` -❯ scx_rustland --monitor 5 -[RustLand] tasks -> r: 1/4 w: 3 /3 | pf: 0 | dispatch -> u: 4 k: 0 c: 0 b: 0 f: 0 | cg: 0 -[RustLand] tasks -> r: 1/4 w: 2 /2 | pf: 0 | dispatch -> u: 28385 k: 0 c: 0 b: 0 f: 0 | cg: 0 -[RustLand] tasks -> r: 0/4 w: 4 /0 | pf: 0 | dispatch -> u: 25288 k: 0 c: 0 b: 0 f: 0 | cg: 0 -[RustLand] tasks -> r: 0/4 w: 2 /0 | pf: 0 | dispatch -> u: 30580 k: 0 c: 0 b: 0 f: 0 | cg: 0 -[RustLand] tasks -> r: 0/4 w: 2 /0 | pf: 0 | dispatch -> u: 30824 k: 0 c: 0 b: 0 f: 0 | cg: 0 -[RustLand] tasks -> r: 1/4 w: 1 /1 | pf: 0 | dispatch -> u: 33178 k: 0 c: 0 b: 0 f: 0 | cg: 0 -``` - -## systemd services - -See: [services](services/README.md) - -## Kernel Feature Status - -The kernel feature is not yet upstream and can be found in the -[sched_ext](https://github.com/sched-ext/sched_ext) repository. The -followings are important branches: - -- [`sched_ext`](https://github.com/sched-ext/sched_ext): The main development - branch. This branch periodically pulls from the - [bpf-next](https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/) - tree to stay in sync with the kernel and BPF developments. -- `sched_ext-release-*`: sched_ext backports on top of released kernels. We - plan to maintain backports for a few recent kernel releases until - sched_ext is merged upstream. Currently maintained backports: - - [`sched_ext-release-v6.6`](https://github.com/sched-ext/sched_ext/tree/sched_ext-release-v6.6) -- `sched_ext-vN`: Patchsets posted upstream. The v4 LKML thread has - high-level discussions. - - [RFC](https://github.com/htejun/sched_ext): - [LMKL thread](http://lkml.kernel.org/r/20221130082313.3241517-1-tj@kernel.org) - - [`sched_ext-v2'](https://github.com/sched-ext/sched_ext/tree/sched_ext-v2): - [LKML thread](http://lkml.kernel.org/r/20230128001639.3510083-1-tj@kernel.org) - - [`sched_ext-v3'](https://github.com/sched-ext/sched_ext/tree/sched_ext-v3): - [LKML thread](http://lkml.kernel.org/r/20230317213333.2174969-1-tj@kernel.org) - - [`sched_ext-v4'](https://github.com/sched-ext/sched_ext/tree/sched_ext-v4): - [LKML thread](http://lkml.kernel.org/r/20230711011412.100319-1-tj@kernel.org) - - [`sched_ext-v5'](https://github.com/sched-ext/sched_ext/tree/sched_ext-v5): - [LKML thread](http://lkml.kernel.org/r/20231111024835.2164816-1-tj@kernel.org) - -## [Breaking Changes](./BREAKING_CHANGES.md) - -[A list of the breaking changes](./BREAKING_CHANGES.md) in the sched_ext kernel tree and the associated commits for the schedulers in this repo. - -## [Developer Guide](./DEVELOPER_GUIDE.md) - -Want to learn how to develop a scheduler or find some useful tools for working -with schedulers? See the developer guide for more details. - -## Getting in Touch - -We aim to build a friendly and approachable community around sched_ext. You -can reach us through the following channels: - -- github: https://github.com/sched-ext/scx -- Slack: [https://schedextworkspace.slack.com](https://join.slack.com/t/schedextworkspace/shared_invite/zt-24c4on3sk-sHlozdLfCZBODfwU6t6dbw) -- Reddit: https://reddit.com/r/sched_ext - -We also hold weekly office hours every Tuesday. Please see the #office-hours -channel on slack for details. To join the slack community, you can use [this -link](https://bit.ly/scx_slack). - -## Additional Resources - -There are blog posts and articles about sched_ext, which helps you to explore -sched_ext in various ways. Followings are some examples: - -- [LWN: The extensible scheduler class (February, 2023)](https://lwn.net/Articles/922405/) -- [arighi's blog: Implement your own kernel CPU scheduler in Ubuntu with sched-ext (July, 2023)](https://arighi.blogspot.com/2023/07/implement-your-own-cpu-scheduler-in.html) -- [Changwoo's blog: sched_ext: a BPF-extensible scheduler class (Part 1) (December, 2023)](https://blogs.igalia.com/changwoo/sched-ext-a-bpf-extensible-scheduler-class-part-1/) -- [arighi's blog: Getting started with sched-ext development (April, 2024)](https://arighi.blogspot.com/2024/04/getting-started-with-sched-ext.html) -- [Changwoo's blog: sched_ext: scheduler architecture and interfaces (Part 2) (June, 2024)](https://blogs.igalia.com/changwoo/sched-ext-scheduler-architecture-and-interfaces-part-2/) -- [arighi's YT channel: scx_bpfland Linux scheduler demo: topology awareness (August, 2024)](https://youtu.be/R-FEZOveG-I) +*`main` will be updated periodically (roughly coinciding with releases of scx), backports will be updated/created on an as-needed basis.* diff --git a/backport-scripts/Dockerfile b/backport-scripts/Dockerfile new file mode 100644 index 000000000..74bf3d5af --- /dev/null +++ b/backport-scripts/Dockerfile @@ -0,0 +1,23 @@ +FROM ubuntu:24.04 +# keep codename and ver in sync +ENV UBUNTU_CODENAME=noble + +ENV DEBIAN_FRONTEND=noninteractive +RUN echo "deb-src http://archive.ubuntu.com/ubuntu ${UBUNTU_CODENAME} main" >> /etc/apt/sources.list +RUN echo "deb-src http://archive.ubuntu.com/ubuntu ${UBUNTU_CODENAME}-updates main" >> /etc/apt/sources.list +RUN cat /etc/apt/sources.list +RUN apt update && apt install -y build-essential \ + libelf-dev \ + libz-dev \ + libcap-dev \ + clang llvm llvm-dev lld \ + binutils-dev \ + pkg-config \ + git \ + python3-pip \ + --no-install-recommends && \ + apt -y build-dep linux --no-install-recommends && \ + apt clean && rm -rf /var/lib/apt/lists/* +RUN pip3 install virtme-ng --break-system-packages +COPY .github/workflows/sched-ext.config /sched-ext.config +COPY backport-scripts/exec-entrypoint.sh /exec-entrypoint.sh diff --git a/backport-scripts/exec-entrypoint.sh b/backport-scripts/exec-entrypoint.sh new file mode 100755 index 000000000..863a630dc --- /dev/null +++ b/backport-scripts/exec-entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -euxo pipefail + +cd / + +if [ -f /.dockerenv ]; then + echo "Running artifact build in docker container"; +else + echo "This script is not being ran inside a docker container, please run this script inside a docker container."; +fi + +git clone --recurse-submodules https://github.com/libbpf/bpftool.git /bpftool + +cd /bpftool/src + +make -j "$(nproc)" + +make -j "$(nproc)" install + +cd / + +SHORT_SHA="$1" +BRANCH="$2" +REPO="$3" + +git clone -b "$BRANCH" "$REPO" /sched-ext-linux + +cd /sched-ext-linux + +git checkout "$SHORT_SHA" + +# this is for backports and mixing new bpf with old kernel +find . -type f -exec sed -i 's/-Werror/-Wno-error/g' {} \; + +vng -v --kconfig --config /sched-ext.config + +make -j "$(nproc)" + +bpftool btf dump file "/sched-ext-linux/vmlinux" format c > "/vmlinux-${SHORT_SHA}.h" + +echo "generated vmlinux-${SHORT_SHA}.h" diff --git a/backport-scripts/generate-backport-vmlinux.sh b/backport-scripts/generate-backport-vmlinux.sh new file mode 100755 index 000000000..bd796d710 --- /dev/null +++ b/backport-scripts/generate-backport-vmlinux.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -euxo pipefail + +SHORT_SHA="$1" +BRANCH="$2" +REPO="$3" + +START_PWD="$(pwd)" + +REPO_ROOT="$(git rev-parse --show-toplevel)" + +docker build -f backport-scripts/Dockerfile -t local-build-container . + +BUILD_CONTAINER="$(docker run -it --rm --detach local-build-container)" + +docker exec -it "${BUILD_CONTAINER}" "/exec-entrypoint.sh" "${SHORT_SHA}" "${BRANCH}" "${REPO:=https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git}" + +clean_up_container () { + ARG=$? + docker rm -f "${BUILD_CONTAINER}" + exit $ARG +} +trap clean_up_container EXIT + +# copy artifact to host +docker cp "${BUILD_CONTAINER}:/vmlinux-${SHORT_SHA}.h" "${REPO_ROOT}/scheds/include/vmlinux/vmlinux-${SHORT_SHA}.h" + +# update vmlinux.h symlink +ln -sf "${REPO_ROOT}/scheds/include/vmlinux/vmlinux-${SHORT_SHA}.h" "${REPO_ROOT}/scheds/include/vmlinux/vmlinux.h" + +# return to where called from +cd "${START_PWD}" + +echo "updated vmlinux.h for kernel version ${SHORT_SHA}" + +exit 0