Skip to content

Commit

Permalink
Merge pull request #735 from hodgesds/layered-weights
Browse files Browse the repository at this point in the history 
scx_layered: Add per layer weights
  • Loading branch information
@hodgesds
hodgesds authored Oct 4, 2024
2 parents 0eb0a5c + c56e60b commit fdc5640
Show file tree
Hide file tree
Showing 4 changed files with 328 additions and 50 deletions.
27 changes: 19 additions & 8 deletions scheds/rust/scx_layered/src/bpf/intf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ enum consts {
MAX_COMM = 16,
MAX_LAYER_MATCH_ORS = 32,
MAX_LAYERS = 16,
MAX_LAYER_WEIGHT = 10000,
MIN_LAYER_WEIGHT = 1,
DEFAULT_LAYER_WEIGHT = 100,
USAGE_HALF_LIFE = 100000000, /* 100ms */

HI_FALLBACK_DSQ_BASE = MAX_LAYERS * MAX_LLCS,
Expand Down Expand Up @@ -150,14 +153,21 @@ struct layer_match_ands {
};

enum layer_growth_algo {
STICKY,
LINEAR,
REVERSE,
RANDOM,
TOPO,
ROUND_ROBIN,
BIG_LITTLE,
LITTLE_BIG,
STICKY,
LINEAR,
REVERSE,
RANDOM,
TOPO,
ROUND_ROBIN,
BIG_LITTLE,
LITTLE_BIG,
};

enum dsq_iter_algo {
DSQ_ITER_LINEAR,
DSQ_ITER_ROUND_ROBIN,
DSQ_ITER_WEIGHT,
DSQ_ITER_REVERSE_WEIGHT,
};

struct layer {
Expand All @@ -168,6 +178,7 @@ struct layer {
u64 max_exec_ns;
u64 yield_step_ns;
u64 slice_ns;
u32 weight;
bool open;
bool preempt;
bool preempt_first;
Expand Down
82 changes: 70 additions & 12 deletions scheds/rust/scx_layered/src/bpf/main.bpf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,11 @@ const volatile u32 nr_llcs = 32; /* !0 for veristat, set during init */
const volatile bool smt_enabled = true;
const volatile bool disable_topology = false;
const volatile bool xnuma_preemption = false;
const volatile bool layer_weight_dsq_iter = false;
const volatile s32 __sibling_cpu[MAX_CPUS];
const volatile unsigned char all_cpus[MAX_CPUS_U8];
const volatile u32 layer_iteration_order[MAX_LAYERS];
const volatile u32 dsq_iter_algo = DSQ_ITER_ROUND_ROBIN;

private(all_cpumask) struct bpf_cpumask __kptr *all_cpumask;
struct layer layers[MAX_LAYERS];
Expand Down Expand Up @@ -79,7 +82,15 @@ static u32 cpu_ctx_layer_idx_inc(struct cpu_ctx *cctx)
return cctx->layer_idx;
}

static __noinline u32 iter_layer_cpu_ctx(u32 layer_idx, int idx)
/*
* Returns the iterator index of a layer ordered by weight.
*/
static u32 dsq_iter_weight_ctx(int idx)
{
return *MEMBER_VPTR(layer_iteration_order, [idx]);
}

static u32 dsq_iter_rr_cpu_ctx(u32 layer_idx, int idx)
{
u32 offset;

Expand All @@ -101,6 +112,33 @@ static __noinline u32 iter_layer_cpu_ctx(u32 layer_idx, int idx)
return offset;
}

/*
* Returns the iterator context for iterating over DSQs using the configured
* DSQ iterator algorithm.
*/
static u32 iter_layer_dsq_ctx(int idx, u32 layer_idx)
{
switch (dsq_iter_algo) {
case DSQ_ITER_LINEAR: {
return idx;
}
case DSQ_ITER_ROUND_ROBIN: {
return dsq_iter_rr_cpu_ctx(layer_idx, idx);
}
case DSQ_ITER_WEIGHT: {
return dsq_iter_weight_ctx(idx);
}
case DSQ_ITER_REVERSE_WEIGHT: {
// TODO: Correctly implement this algo, see:
// https://github.com/sched-ext/scx/issues/738
return dsq_iter_weight_ctx(idx);
}
default:
scx_bpf_error("unknown dsq iter algo");
return 0;
}
}

// return the dsq id for the layer based on the LLC id.
static __noinline u64 layer_dsq_id(u32 layer_id, u32 llc_id)
{
Expand Down Expand Up @@ -1160,14 +1198,15 @@ void BPF_STRUCT_OPS(layered_dispatch, s32 cpu, struct task_struct *prev)

/* consume preempting layers first */
bpf_for(idx, 0, nr_layers) {
layer_idx = iter_layer_cpu_ctx(cctx->layer_idx, idx);
if (disable_topology) {
if (MEMBER_VPTR(layers, [layer_idx].preempt) && scx_bpf_consume(layer_idx))
if (MEMBER_VPTR(layers, [idx].preempt) && scx_bpf_consume(idx))
return;
} else {
layer_idx = iter_layer_dsq_ctx(idx, cctx->layer_idx);
bpf_for(llc_id, 0, nr_llcs) {
dsq_id = layer_dsq_id(layer_idx, llc_id);
if (MEMBER_VPTR(layers, [layer_idx].preempt) && scx_bpf_consume(dsq_id))
if (MEMBER_VPTR(layers, [layer_idx].preempt) &&
scx_bpf_consume(dsq_id))
return;
}
}
Expand All @@ -1179,21 +1218,22 @@ void BPF_STRUCT_OPS(layered_dispatch, s32 cpu, struct task_struct *prev)

/* consume !open layers second */
bpf_for(idx, 0, nr_layers) {
layer_idx = iter_layer_cpu_ctx(cctx->layer_idx, idx);
if (disable_topology) {
struct layer *layer = &layers[layer_idx];
layer_idx = idx;
struct layer *layer = &layers[idx];
struct cpumask *layer_cpumask;

/* consume matching layers */
if (!(layer_cpumask = lookup_layer_cpumask(layer_idx)))
if (!(layer_cpumask = lookup_layer_cpumask(idx)))
return;

if (bpf_cpumask_test_cpu(cpu, layer_cpumask) ||
(cpu == fallback_cpu && layer->nr_cpus == 0)) {
if (scx_bpf_consume(layer_idx))
if (scx_bpf_consume(idx))
return;
}
} else {
layer_idx = iter_layer_dsq_ctx(idx, cctx->layer_idx);
bpf_for(llc_id, 0, nr_llcs) {
struct layer *layer = &layers[layer_idx];
struct cpumask *layer_cpumask;
Expand All @@ -1205,7 +1245,7 @@ void BPF_STRUCT_OPS(layered_dispatch, s32 cpu, struct task_struct *prev)

if (bpf_cpumask_test_cpu(cpu, layer_cpumask) ||
(cpu <= nr_possible_cpus && cpu == fallback_cpu &&
layer->nr_cpus == 0)) {
MEMBER_VPTR(layer, ->nr_cpus) == 0)) {
if (scx_bpf_consume(dsq_id))
return;
}
Expand All @@ -1215,12 +1255,12 @@ void BPF_STRUCT_OPS(layered_dispatch, s32 cpu, struct task_struct *prev)

/* consume !preempting open layers */
bpf_for(idx, 0, nr_layers) {
layer_idx = iter_layer_cpu_ctx(cctx->layer_idx, idx);
if (disable_topology) {
if (!layers[layer_idx].preempt && layers[layer_idx].open &&
scx_bpf_consume(layer_idx))
if (!layers[idx].preempt && layers[idx].open &&
scx_bpf_consume(idx))
return;
} else {
layer_idx = iter_layer_dsq_ctx(idx, cctx->layer_idx);
bpf_for(llc_id, 0, nr_llcs) {
dsq_id = layer_dsq_id(layer_idx, llc_id);

Expand Down Expand Up @@ -1885,6 +1925,22 @@ void BPF_STRUCT_OPS(layered_dump, struct scx_dump_ctx *dctx)
dsq_first_runnable_for_ms(LO_FALLBACK_DSQ, now));
}

static void print_iter_order() {
int i;
struct cpu_ctx *cctx;

if (!(cctx = lookup_cpu_ctx(-1))) {
scx_bpf_error("failed to get cpu ctx");
return;
}

trace("ITER algo: %d", dsq_iter_algo);
bpf_for(i, 0, nr_layers) {
trace("ITER order i: %d %d\n", i,
iter_layer_dsq_ctx(i, cctx->layer_idx));
}
}

s32 BPF_STRUCT_OPS_SLEEPABLE(layered_init)
{
struct bpf_cpumask *cpumask;
Expand Down Expand Up @@ -2059,6 +2115,8 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(layered_init)
}
}

print_iter_order();

return 0;
}

Expand Down
Loading

0 comments on commit fdc5640

Please sign in to comment.