all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubst %.c,%.skel.h,$(prog)))
auto-test-targets := \
+ consume_immed \
create_dsq \
dequeue \
+ dsq_reenq \
+ enq_immed \
enq_last_no_enq_fails \
ddsp_bogus_dsq_fail \
ddsp_vtimelocal_fail \
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Validate SCX_ENQ_IMMED semantics through the consume path.
+ *
+ * This is the orthogonal counterpart to enq_immed:
+ *
+ * enq_immed: SCX_ENQ_IMMED via scx_bpf_dsq_insert() to local DSQ
+ * with SCX_OPS_ALWAYS_ENQ_IMMED
+ *
+ * consume_immed: SCX_ENQ_IMMED via scx_bpf_dsq_move_to_local() with
+ * explicit SCX_ENQ_IMMED in enq_flags (requires v2 kfunc)
+ *
+ * Worker threads belonging to test_tgid are inserted into USER_DSQ.
+ * ops.dispatch() on CPU 0 consumes from USER_DSQ with SCX_ENQ_IMMED.
+ * With multiple workers competing for CPU 0, dsq->nr > 1 triggers the
+ * IMMED slow path (reenqueue with SCX_TASK_REENQ_IMMED).
+ *
+ * Requires scx_bpf_dsq_move_to_local___v2() (v7.1+) for enq_flags support.
+ */
+
+#include <scx/common.bpf.h>
+
+char _license[] SEC("license") = "GPL";
+
+UEI_DEFINE(uei);
+
+#define USER_DSQ 0
+
+/* Set by userspace to identify the test process group. */
+const volatile u32 test_tgid;
+
+/*
+ * SCX_TASK_REENQ_REASON_MASK and SCX_TASK_REENQ_IMMED are exported via
+ * vmlinux BTF as part of enum scx_ent_flags.
+ */
+
+u64 nr_consume_immed_reenq;
+
+void BPF_STRUCT_OPS(consume_immed_enqueue, struct task_struct *p,
+ u64 enq_flags)
+{
+ if (enq_flags & SCX_ENQ_REENQ) {
+ u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK;
+
+ if (reason == SCX_TASK_REENQ_IMMED)
+ __sync_fetch_and_add(&nr_consume_immed_reenq, 1);
+ }
+
+ if (p->tgid == (pid_t)test_tgid)
+ scx_bpf_dsq_insert(p, USER_DSQ, SCX_SLICE_DFL, enq_flags);
+ else
+ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL,
+ enq_flags);
+}
+
+void BPF_STRUCT_OPS(consume_immed_dispatch, s32 cpu, struct task_struct *prev)
+{
+ if (cpu == 0)
+ scx_bpf_dsq_move_to_local(USER_DSQ, SCX_ENQ_IMMED);
+ else
+ scx_bpf_dsq_move_to_local(SCX_DSQ_GLOBAL, 0);
+}
+
+s32 BPF_STRUCT_OPS_SLEEPABLE(consume_immed_init)
+{
+ /*
+ * scx_bpf_dsq_move_to_local___v2() adds the enq_flags parameter.
+ * On older kernels the consume path cannot pass SCX_ENQ_IMMED.
+ */
+ if (!bpf_ksym_exists(scx_bpf_dsq_move_to_local___v2)) {
+ scx_bpf_error("scx_bpf_dsq_move_to_local v2 not available");
+ return -EOPNOTSUPP;
+ }
+
+ return scx_bpf_create_dsq(USER_DSQ, -1);
+}
+
+void BPF_STRUCT_OPS(consume_immed_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SCX_OPS_DEFINE(consume_immed_ops,
+ .enqueue = (void *)consume_immed_enqueue,
+ .dispatch = (void *)consume_immed_dispatch,
+ .init = (void *)consume_immed_init,
+ .exit = (void *)consume_immed_exit,
+ .name = "consume_immed")
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Userspace test for SCX_ENQ_IMMED via the consume path.
+ *
+ * Validates that scx_bpf_dsq_move_to_local(USER_DSQ, SCX_ENQ_IMMED) on
+ * a busy CPU triggers the IMMED slow path, re-enqueuing tasks through
+ * ops.enqueue() with SCX_TASK_REENQ_IMMED.
+ *
+ * Skipped on single-CPU systems where local DSQ contention cannot occur.
+ */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <bpf/bpf.h>
+#include <scx/common.h>
+#include "consume_immed.bpf.skel.h"
+#include "scx_test.h"
+
+#define NUM_WORKERS 4
+#define TEST_DURATION_SEC 3
+
+static volatile bool stop_workers;
+
+static void *worker_fn(void *arg)
+{
+ while (!stop_workers) {
+ volatile unsigned long i;
+
+ for (i = 0; i < 100000UL; i++)
+ ;
+ usleep(100);
+ }
+ return NULL;
+}
+
+static enum scx_test_status setup(void **ctx)
+{
+ struct consume_immed *skel;
+
+ if (!__COMPAT_has_ksym("scx_bpf_dsq_move_to_local___v2")) {
+ fprintf(stderr,
+ "SKIP: scx_bpf_dsq_move_to_local v2 not available\n");
+ return SCX_TEST_SKIP;
+ }
+
+ skel = consume_immed__open();
+ SCX_FAIL_IF(!skel, "Failed to open");
+ SCX_ENUM_INIT(skel);
+
+ skel->rodata->test_tgid = (u32)getpid();
+
+ SCX_FAIL_IF(consume_immed__load(skel), "Failed to load skel");
+
+ *ctx = skel;
+ return SCX_TEST_PASS;
+}
+
+static enum scx_test_status run(void *ctx)
+{
+ struct consume_immed *skel = ctx;
+ struct bpf_link *link;
+ pthread_t workers[NUM_WORKERS];
+ long nproc;
+ int i;
+ u64 reenq;
+
+ nproc = sysconf(_SC_NPROCESSORS_ONLN);
+ if (nproc <= 1) {
+ fprintf(stderr,
+ "SKIP: single CPU, consume IMMED slow path may not trigger\n");
+ return SCX_TEST_SKIP;
+ }
+
+ link = bpf_map__attach_struct_ops(skel->maps.consume_immed_ops);
+ SCX_FAIL_IF(!link, "Failed to attach scheduler");
+
+ stop_workers = false;
+ for (i = 0; i < NUM_WORKERS; i++) {
+ SCX_FAIL_IF(pthread_create(&workers[i], NULL, worker_fn, NULL),
+ "Failed to create worker %d", i);
+ }
+
+ sleep(TEST_DURATION_SEC);
+
+ reenq = skel->bss->nr_consume_immed_reenq;
+
+ stop_workers = true;
+ for (i = 0; i < NUM_WORKERS; i++)
+ pthread_join(workers[i], NULL);
+
+ bpf_link__destroy(link);
+
+ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG));
+ SCX_GT(reenq, 0);
+
+ return SCX_TEST_PASS;
+}
+
+static void cleanup(void *ctx)
+{
+ struct consume_immed *skel = ctx;
+
+ consume_immed__destroy(skel);
+}
+
+struct scx_test consume_immed = {
+ .name = "consume_immed",
+ .description = "Verify SCX_ENQ_IMMED slow path via "
+ "scx_bpf_dsq_move_to_local() consume path",
+ .setup = setup,
+ .run = run,
+ .cleanup = cleanup,
+};
+REGISTER_SCX_TEST(&consume_immed)
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Validate scx_bpf_dsq_reenq() semantics on user DSQs.
+ *
+ * A BPF timer periodically calls scx_bpf_dsq_reenq() on a user DSQ,
+ * causing tasks to be re-enqueued through ops.enqueue() with SCX_ENQ_REENQ
+ * set and SCX_TASK_REENQ_KFUNC recorded in p->scx.flags.
+ *
+ * The test verifies:
+ * - scx_bpf_dsq_reenq() triggers ops.enqueue() with SCX_ENQ_REENQ
+ * - The reenqueue reason is SCX_TASK_REENQ_KFUNC (bit 12 set)
+ * - Tasks are correctly re-dispatched after reenqueue
+ */
+
+#include <scx/common.bpf.h>
+
+char _license[] SEC("license") = "GPL";
+
+UEI_DEFINE(uei);
+
+#define USER_DSQ 0
+
+/*
+ * SCX_TASK_REENQ_REASON_MASK and SCX_TASK_REENQ_KFUNC are exported via
+ * vmlinux BTF as part of enum scx_ent_flags.
+ */
+
+/* 5ms timer interval */
+#define REENQ_TIMER_NS (5 * 1000 * 1000ULL)
+
+/*
+ * Number of times ops.enqueue() was called with SCX_ENQ_REENQ set and
+ * SCX_TASK_REENQ_KFUNC recorded in p->scx.flags.
+ */
+u64 nr_reenq_kfunc;
+
+struct reenq_timer_val {
+ struct bpf_timer timer;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, u32);
+ __type(value, struct reenq_timer_val);
+} reenq_timer SEC(".maps");
+
+/*
+ * Timer callback: reenqueue all tasks currently sitting on USER_DSQ back
+ * through ops.enqueue() with SCX_ENQ_REENQ | SCX_TASK_REENQ_KFUNC.
+ */
+static int reenq_timerfn(void *map, int *key, struct bpf_timer *timer)
+{
+ scx_bpf_dsq_reenq(USER_DSQ, 0);
+ bpf_timer_start(timer, REENQ_TIMER_NS, 0);
+ return 0;
+}
+
+void BPF_STRUCT_OPS(dsq_reenq_enqueue, struct task_struct *p, u64 enq_flags)
+{
+ /*
+ * If this is a kfunc-triggered reenqueue, verify that
+ * SCX_TASK_REENQ_KFUNC is recorded in p->scx.flags.
+ */
+ if (enq_flags & SCX_ENQ_REENQ) {
+ u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK;
+
+ if (reason == SCX_TASK_REENQ_KFUNC)
+ __sync_fetch_and_add(&nr_reenq_kfunc, 1);
+ }
+
+ /*
+ * Always dispatch to USER_DSQ so the timer can reenqueue tasks again
+ * on the next tick.
+ */
+ scx_bpf_dsq_insert(p, USER_DSQ, SCX_SLICE_DFL, enq_flags);
+}
+
+void BPF_STRUCT_OPS(dsq_reenq_dispatch, s32 cpu, struct task_struct *prev)
+{
+ scx_bpf_dsq_move_to_local(USER_DSQ, 0);
+}
+
+s32 BPF_STRUCT_OPS_SLEEPABLE(dsq_reenq_init)
+{
+ struct reenq_timer_val *tval;
+ u32 key = 0;
+ s32 ret;
+
+ ret = scx_bpf_create_dsq(USER_DSQ, -1);
+ if (ret)
+ return ret;
+
+ if (!__COMPAT_has_generic_reenq()) {
+ scx_bpf_error("scx_bpf_dsq_reenq() not available");
+ return -EOPNOTSUPP;
+ }
+
+ tval = bpf_map_lookup_elem(&reenq_timer, &key);
+ if (!tval)
+ return -ESRCH;
+
+ bpf_timer_init(&tval->timer, &reenq_timer, CLOCK_MONOTONIC);
+ bpf_timer_set_callback(&tval->timer, reenq_timerfn);
+
+ return bpf_timer_start(&tval->timer, REENQ_TIMER_NS, 0);
+}
+
+void BPF_STRUCT_OPS(dsq_reenq_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SCX_OPS_DEFINE(dsq_reenq_ops,
+ .enqueue = (void *)dsq_reenq_enqueue,
+ .dispatch = (void *)dsq_reenq_dispatch,
+ .init = (void *)dsq_reenq_init,
+ .exit = (void *)dsq_reenq_exit,
+ .timeout_ms = 10000,
+ .name = "dsq_reenq")
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Userspace test for scx_bpf_dsq_reenq() semantics.
+ *
+ * Attaches the dsq_reenq BPF scheduler, runs workload threads that
+ * sleep and yield to keep tasks on USER_DSQ, waits for the BPF timer
+ * to fire several times, then verifies that at least one kfunc-triggered
+ * reenqueue was observed (ops.enqueue() called with SCX_ENQ_REENQ and
+ * SCX_TASK_REENQ_KFUNC in p->scx.flags).
+ */
+#include <bpf/bpf.h>
+#include <scx/common.h>
+#include <unistd.h>
+#include <pthread.h>
+#include "dsq_reenq.bpf.skel.h"
+#include "scx_test.h"
+
+#define NUM_WORKERS 4
+#define TEST_DURATION_SEC 3
+
+static volatile bool stop_workers;
+static pthread_t workers[NUM_WORKERS];
+
+static void *worker_fn(void *arg)
+{
+ while (!stop_workers) {
+ usleep(500);
+ sched_yield();
+ }
+ return NULL;
+}
+
+static enum scx_test_status setup(void **ctx)
+{
+ struct dsq_reenq *skel;
+
+ if (!__COMPAT_has_ksym("scx_bpf_dsq_reenq")) {
+ fprintf(stderr, "SKIP: scx_bpf_dsq_reenq() not available\n");
+ return SCX_TEST_SKIP;
+ }
+
+ skel = dsq_reenq__open();
+ SCX_FAIL_IF(!skel, "Failed to open");
+ SCX_ENUM_INIT(skel);
+ SCX_FAIL_IF(dsq_reenq__load(skel), "Failed to load skel");
+
+ *ctx = skel;
+ return SCX_TEST_PASS;
+}
+
+static enum scx_test_status run(void *ctx)
+{
+ struct dsq_reenq *skel = ctx;
+ struct bpf_link *link;
+ int i;
+
+ link = bpf_map__attach_struct_ops(skel->maps.dsq_reenq_ops);
+ SCX_FAIL_IF(!link, "Failed to attach scheduler");
+
+ stop_workers = false;
+ for (i = 0; i < NUM_WORKERS; i++) {
+ SCX_FAIL_IF(pthread_create(&workers[i], NULL, worker_fn, NULL),
+ "Failed to create worker %d", i);
+ }
+
+ sleep(TEST_DURATION_SEC);
+
+ stop_workers = true;
+ for (i = 0; i < NUM_WORKERS; i++)
+ pthread_join(workers[i], NULL);
+
+ bpf_link__destroy(link);
+
+ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG));
+ SCX_GT(skel->bss->nr_reenq_kfunc, 0);
+
+ return SCX_TEST_PASS;
+}
+
+static void cleanup(void *ctx)
+{
+ struct dsq_reenq *skel = ctx;
+
+ dsq_reenq__destroy(skel);
+}
+
+struct scx_test dsq_reenq = {
+ .name = "dsq_reenq",
+ .description = "Verify scx_bpf_dsq_reenq() triggers enqueue with "
+ "SCX_ENQ_REENQ and SCX_TASK_REENQ_KFUNC reason",
+ .setup = setup,
+ .run = run,
+ .cleanup = cleanup,
+};
+REGISTER_SCX_TEST(&dsq_reenq)
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Validate SCX_ENQ_IMMED fast/slow path semantics via the direct insert path.
+ *
+ * With SCX_OPS_ALWAYS_ENQ_IMMED set, the kernel automatically adds
+ * SCX_ENQ_IMMED to every local DSQ dispatch. When the target CPU's local
+ * DSQ already has tasks queued (dsq->nr > 1), the kernel re-enqueues the
+ * task through ops.enqueue() with SCX_ENQ_REENQ and SCX_TASK_REENQ_IMMED
+ * recorded in p->scx.flags (the "slow path").
+ *
+ * Worker threads are pinned to CPU 0 via SCX_DSQ_LOCAL_ON to guarantee
+ * local DSQ contention.
+ */
+
+#include <scx/common.bpf.h>
+
+char _license[] SEC("license") = "GPL";
+
+UEI_DEFINE(uei);
+
+/* Set by userspace to identify the test process group. */
+const volatile u32 test_tgid;
+
+/*
+ * SCX_TASK_REENQ_REASON_MASK and SCX_TASK_REENQ_IMMED are exported via
+ * vmlinux BTF as part of enum scx_ent_flags.
+ */
+
+u64 nr_immed_reenq;
+
+void BPF_STRUCT_OPS(enq_immed_enqueue, struct task_struct *p, u64 enq_flags)
+{
+ if (enq_flags & SCX_ENQ_REENQ) {
+ u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK;
+
+ if (reason == SCX_TASK_REENQ_IMMED)
+ __sync_fetch_and_add(&nr_immed_reenq, 1);
+ }
+
+ if (p->tgid == (pid_t)test_tgid)
+ scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | 0, SCX_SLICE_DFL,
+ enq_flags);
+ else
+ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL,
+ enq_flags);
+}
+
+void BPF_STRUCT_OPS(enq_immed_dispatch, s32 cpu, struct task_struct *prev)
+{
+ scx_bpf_dsq_move_to_local(SCX_DSQ_GLOBAL, 0);
+}
+
+void BPF_STRUCT_OPS(enq_immed_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SCX_OPS_DEFINE(enq_immed_ops,
+ .enqueue = (void *)enq_immed_enqueue,
+ .dispatch = (void *)enq_immed_dispatch,
+ .exit = (void *)enq_immed_exit,
+ .flags = SCX_OPS_ALWAYS_ENQ_IMMED,
+ .name = "enq_immed")
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Userspace test for SCX_ENQ_IMMED via the direct insert path.
+ *
+ * Validates that dispatching tasks to a busy CPU's local DSQ with
+ * SCX_OPS_ALWAYS_ENQ_IMMED triggers the IMMED slow path: the kernel
+ * re-enqueues the task through ops.enqueue() with SCX_TASK_REENQ_IMMED.
+ *
+ * Skipped on single-CPU systems where local DSQ contention cannot occur.
+ */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <bpf/bpf.h>
+#include <scx/common.h>
+#include "enq_immed.bpf.skel.h"
+#include "scx_test.h"
+
+#define NUM_WORKERS 4
+#define TEST_DURATION_SEC 3
+
+static volatile bool stop_workers;
+
+static void *worker_fn(void *arg)
+{
+ while (!stop_workers) {
+ volatile unsigned long i;
+
+ for (i = 0; i < 100000UL; i++)
+ ;
+ usleep(100);
+ }
+ return NULL;
+}
+
+static enum scx_test_status setup(void **ctx)
+{
+ struct enq_immed *skel;
+ u64 v;
+
+ if (!__COMPAT_read_enum("scx_ops_flags",
+ "SCX_OPS_ALWAYS_ENQ_IMMED", &v)) {
+ fprintf(stderr,
+ "SKIP: SCX_OPS_ALWAYS_ENQ_IMMED not available\n");
+ return SCX_TEST_SKIP;
+ }
+
+ skel = enq_immed__open();
+ SCX_FAIL_IF(!skel, "Failed to open");
+ SCX_ENUM_INIT(skel);
+
+ skel->rodata->test_tgid = (u32)getpid();
+
+ SCX_FAIL_IF(enq_immed__load(skel), "Failed to load skel");
+
+ *ctx = skel;
+ return SCX_TEST_PASS;
+}
+
+static enum scx_test_status run(void *ctx)
+{
+ struct enq_immed *skel = ctx;
+ struct bpf_link *link;
+ pthread_t workers[NUM_WORKERS];
+ long nproc;
+ int i;
+ u64 reenq;
+
+ nproc = sysconf(_SC_NPROCESSORS_ONLN);
+ if (nproc <= 1) {
+ fprintf(stderr,
+ "SKIP: single CPU, IMMED slow path may not trigger\n");
+ return SCX_TEST_SKIP;
+ }
+
+ link = bpf_map__attach_struct_ops(skel->maps.enq_immed_ops);
+ SCX_FAIL_IF(!link, "Failed to attach scheduler");
+
+ stop_workers = false;
+ for (i = 0; i < NUM_WORKERS; i++) {
+ SCX_FAIL_IF(pthread_create(&workers[i], NULL, worker_fn, NULL),
+ "Failed to create worker %d", i);
+ }
+
+ sleep(TEST_DURATION_SEC);
+
+ reenq = skel->bss->nr_immed_reenq;
+
+ stop_workers = true;
+ for (i = 0; i < NUM_WORKERS; i++)
+ pthread_join(workers[i], NULL);
+
+ bpf_link__destroy(link);
+
+ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG));
+ SCX_GT(reenq, 0);
+
+ return SCX_TEST_PASS;
+}
+
+static void cleanup(void *ctx)
+{
+ struct enq_immed *skel = ctx;
+
+ enq_immed__destroy(skel);
+}
+
+struct scx_test enq_immed = {
+ .name = "enq_immed",
+ .description = "Verify SCX_ENQ_IMMED slow path via direct insert "
+ "with SCX_OPS_ALWAYS_ENQ_IMMED",
+ .setup = setup,
+ .run = run,
+ .cleanup = cleanup,
+};
+REGISTER_SCX_TEST(&enq_immed)
projects / thirdparty / kernel / linux.git / commitdiff
