#ifdef LSP
#define __bpf__
-#include "../vmlinux/vmlinux.h"
+#include "../vmlinux.h"
#else
#include "vmlinux.h"
#endif
#define PF_EXITING 0x00000004
#define CLOCK_MONOTONIC 1
+extern int LINUX_KERNEL_VERSION __kconfig;
+extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak;
+extern const char CONFIG_LOCALVERSION[64] __kconfig __weak;
+
/*
* Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can
* lead to really confusing misbehaviors. Let's trigger a build failure.
_Pragma("GCC diagnostic push") \
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
___bpf_fill(___param, args); \
- _Pragma("GCC diagnostic pop") \
+ _Pragma("GCC diagnostic pop")
/*
* scx_bpf_exit() wraps the scx_bpf_exit_bstr() kfunc with variadic arguments
___scx_bpf_bstr_format_checker(fmt, ##args); \
})
+/*
+ * scx_bpf_dump_header() is a wrapper around scx_bpf_dump that adds a header
+ * of system information for debugging.
+ */
+#define scx_bpf_dump_header() \
+({ \
+ scx_bpf_dump("kernel: %d.%d.%d %s\ncc: %s\n", \
+ LINUX_KERNEL_VERSION >> 16, \
+ LINUX_KERNEL_VERSION >> 8 & 0xFF, \
+ LINUX_KERNEL_VERSION & 0xFF, \
+ CONFIG_LOCALVERSION, \
+ CONFIG_CC_VERSION_TEXT); \
+})
+
#define BPF_STRUCT_OPS(name, args...) \
SEC("struct_ops/"#name) \
BPF_PROG(name, ##args)
const struct cpumask *src2) __ksym;
u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym;
+int bpf_iter_bits_new(struct bpf_iter_bits *it, const u64 *unsafe_ptr__ign, u32 nr_words) __ksym;
+int *bpf_iter_bits_next(struct bpf_iter_bits *it) __ksym;
+void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym;
+
+#define def_iter_struct(name) \
+struct bpf_iter_##name { \
+ struct bpf_iter_bits it; \
+ const struct cpumask *bitmap; \
+};
+
+#define def_iter_new(name) \
+static inline int bpf_iter_##name##_new( \
+ struct bpf_iter_##name *it, const u64 *unsafe_ptr__ign, u32 nr_words) \
+{ \
+ it->bitmap = scx_bpf_get_##name##_cpumask(); \
+ return bpf_iter_bits_new(&it->it, (const u64 *)it->bitmap, \
+ sizeof(struct cpumask) / 8); \
+}
+
+#define def_iter_next(name) \
+static inline int *bpf_iter_##name##_next(struct bpf_iter_##name *it) { \
+ return bpf_iter_bits_next(&it->it); \
+}
+
+#define def_iter_destroy(name) \
+static inline void bpf_iter_##name##_destroy(struct bpf_iter_##name *it) { \
+ scx_bpf_put_cpumask(it->bitmap); \
+ bpf_iter_bits_destroy(&it->it); \
+}
+#define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu)
+
+/// Provides iterator for possible and online cpus.
+///
+/// # Example
+///
+/// ```
+/// static inline void example_use() {
+/// int *cpu;
+///
+/// for_each_possible_cpu(cpu){
+/// bpf_printk("CPU %d is possible", *cpu);
+/// }
+///
+/// for_each_online_cpu(cpu){
+/// bpf_printk("CPU %d is online", *cpu);
+/// }
+/// }
+/// ```
+def_iter_struct(possible);
+def_iter_new(possible);
+def_iter_next(possible);
+def_iter_destroy(possible);
+#define for_each_possible_cpu(cpu) bpf_for_each(possible, cpu, NULL, 0)
+
+def_iter_struct(online);
+def_iter_new(online);
+def_iter_next(online);
+def_iter_destroy(online);
+#define for_each_online_cpu(cpu) bpf_for_each(online, cpu, NULL, 0)
+
/*
* Access a cpumask in read-only mode (typically to check bits).
*/
}
#include "compat.bpf.h"
+#include "enums.bpf.h"
#endif /* __SCX_COMMON_BPF_H */
#include "user_exit_info.h"
#include "compat.h"
+#include "enums.h"
+
+/* not available when building kernel tools/sched_ext */
+#if __has_include(<lib/sdt_task.h>)
+#include <lib/sdt_task.h>
+#endif
#endif /* __SCHED_EXT_COMMON_H */
__skel = __scx_name##__open(); \
SCX_BUG_ON(!__skel, "Could not open " #__scx_name); \
__skel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \
+ SCX_ENUM_INIT(__skel); \
__skel; \
})
--- /dev/null
+/*
+ * WARNING: This file is autogenerated from scripts/gen_enums.py. If you would
+ * like to access an enum that is currently missing, add it to the script
+ * and run it from the root directory to update this file.
+ */
+
+const volatile u64 __SCX_OPS_NAME_LEN __weak;
+#define SCX_OPS_NAME_LEN __SCX_OPS_NAME_LEN
+
+const volatile u64 __SCX_SLICE_DFL __weak;
+#define SCX_SLICE_DFL __SCX_SLICE_DFL
+
+const volatile u64 __SCX_SLICE_INF __weak;
+#define SCX_SLICE_INF __SCX_SLICE_INF
+
+const volatile u64 __SCX_DSQ_FLAG_BUILTIN __weak;
+#define SCX_DSQ_FLAG_BUILTIN __SCX_DSQ_FLAG_BUILTIN
+
+const volatile u64 __SCX_DSQ_FLAG_LOCAL_ON __weak;
+#define SCX_DSQ_FLAG_LOCAL_ON __SCX_DSQ_FLAG_LOCAL_ON
+
+const volatile u64 __SCX_DSQ_INVALID __weak;
+#define SCX_DSQ_INVALID __SCX_DSQ_INVALID
+
+const volatile u64 __SCX_DSQ_GLOBAL __weak;
+#define SCX_DSQ_GLOBAL __SCX_DSQ_GLOBAL
+
+const volatile u64 __SCX_DSQ_LOCAL __weak;
+#define SCX_DSQ_LOCAL __SCX_DSQ_LOCAL
+
+const volatile u64 __SCX_DSQ_LOCAL_ON __weak;
+#define SCX_DSQ_LOCAL_ON __SCX_DSQ_LOCAL_ON
+
+const volatile u64 __SCX_DSQ_LOCAL_CPU_MASK __weak;
+#define SCX_DSQ_LOCAL_CPU_MASK __SCX_DSQ_LOCAL_CPU_MASK
+
+const volatile u64 __SCX_TASK_QUEUED __weak;
+#define SCX_TASK_QUEUED __SCX_TASK_QUEUED
+
+const volatile u64 __SCX_TASK_RESET_RUNNABLE_AT __weak;
+#define SCX_TASK_RESET_RUNNABLE_AT __SCX_TASK_RESET_RUNNABLE_AT
+
+const volatile u64 __SCX_TASK_DEQD_FOR_SLEEP __weak;
+#define SCX_TASK_DEQD_FOR_SLEEP __SCX_TASK_DEQD_FOR_SLEEP
+
+const volatile u64 __SCX_TASK_STATE_SHIFT __weak;
+#define SCX_TASK_STATE_SHIFT __SCX_TASK_STATE_SHIFT
+
+const volatile u64 __SCX_TASK_STATE_BITS __weak;
+#define SCX_TASK_STATE_BITS __SCX_TASK_STATE_BITS
+
+const volatile u64 __SCX_TASK_STATE_MASK __weak;
+#define SCX_TASK_STATE_MASK __SCX_TASK_STATE_MASK
+
+const volatile u64 __SCX_TASK_CURSOR __weak;
+#define SCX_TASK_CURSOR __SCX_TASK_CURSOR
+
+const volatile u64 __SCX_TASK_NONE __weak;
+#define SCX_TASK_NONE __SCX_TASK_NONE
+
+const volatile u64 __SCX_TASK_INIT __weak;
+#define SCX_TASK_INIT __SCX_TASK_INIT
+
+const volatile u64 __SCX_TASK_READY __weak;
+#define SCX_TASK_READY __SCX_TASK_READY
+
+const volatile u64 __SCX_TASK_ENABLED __weak;
+#define SCX_TASK_ENABLED __SCX_TASK_ENABLED
+
+const volatile u64 __SCX_TASK_NR_STATES __weak;
+#define SCX_TASK_NR_STATES __SCX_TASK_NR_STATES
+
+const volatile u64 __SCX_TASK_DSQ_ON_PRIQ __weak;
+#define SCX_TASK_DSQ_ON_PRIQ __SCX_TASK_DSQ_ON_PRIQ
+
+const volatile u64 __SCX_KICK_IDLE __weak;
+#define SCX_KICK_IDLE __SCX_KICK_IDLE
+
+const volatile u64 __SCX_KICK_PREEMPT __weak;
+#define SCX_KICK_PREEMPT __SCX_KICK_PREEMPT
+
+const volatile u64 __SCX_KICK_WAIT __weak;
+#define SCX_KICK_WAIT __SCX_KICK_WAIT
+
+const volatile u64 __SCX_ENQ_WAKEUP __weak;
+#define SCX_ENQ_WAKEUP __SCX_ENQ_WAKEUP
+
+const volatile u64 __SCX_ENQ_HEAD __weak;
+#define SCX_ENQ_HEAD __SCX_ENQ_HEAD
+
+const volatile u64 __SCX_ENQ_PREEMPT __weak;
+#define SCX_ENQ_PREEMPT __SCX_ENQ_PREEMPT
+
+const volatile u64 __SCX_ENQ_REENQ __weak;
+#define SCX_ENQ_REENQ __SCX_ENQ_REENQ
+
+const volatile u64 __SCX_ENQ_LAST __weak;
+#define SCX_ENQ_LAST __SCX_ENQ_LAST
+
+const volatile u64 __SCX_ENQ_CLEAR_OPSS __weak;
+#define SCX_ENQ_CLEAR_OPSS __SCX_ENQ_CLEAR_OPSS
+
+const volatile u64 __SCX_ENQ_DSQ_PRIQ __weak;
+#define SCX_ENQ_DSQ_PRIQ __SCX_ENQ_DSQ_PRIQ
+
--- /dev/null
+/*
+ * WARNING: This file is autogenerated from scripts/gen_enums.py. If you would
+ * like to access an enum that is currently missing, add it to the script
+ * and run it from the root directory to update this file.
+ */
+
+#define SCX_ENUM_INIT(skel) do { \
+ SCX_ENUM_SET(skel, scx_public_consts, SCX_OPS_NAME_LEN); \
+ SCX_ENUM_SET(skel, scx_public_consts, SCX_SLICE_DFL); \
+ SCX_ENUM_SET(skel, scx_public_consts, SCX_SLICE_INF); \
+ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_BUILTIN); \
+ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_LOCAL_ON); \
+ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_INVALID); \
+ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_GLOBAL); \
+ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_LOCAL); \
+ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_LOCAL_ON); \
+ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_LOCAL_CPU_MASK); \
+ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_QUEUED); \
+ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_RESET_RUNNABLE_AT); \
+ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_DEQD_FOR_SLEEP); \
+ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_SHIFT); \
+ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_BITS); \
+ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_MASK); \
+ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_CURSOR); \
+ SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NONE); \
+ SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_INIT); \
+ SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_READY); \
+ SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_ENABLED); \
+ SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NR_STATES); \
+ SCX_ENUM_SET(skel, scx_ent_dsq_flags, SCX_TASK_DSQ_ON_PRIQ); \
+ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_IDLE); \
+ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT); \
+ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_WAIT); \
+ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_WAKEUP); \
+ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_HEAD); \
+ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT); \
+ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \
+ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \
+ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \
+ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_DSQ_PRIQ); \
+} while (0)
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Convenience macros for getting/setting struct scx_enums instances.
+ *
+ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
+ */
+#ifndef __SCX_ENUMS_BPF_H
+#define __SCX_ENUMS_BPF_H
+
+#include "enums.autogen.bpf.h"
+
+#endif /* __SCX_ENUMS_BPF_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Define struct scx_enums that stores the load-time values of enums
+ * used by the BPF program.
+ *
+ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
+ */
+
+#ifndef __SCX_ENUMS_H
+#define __SCX_ENUMS_H
+
+static inline void __ENUM_set(u64 *val, char *type, char *name)
+{
+ bool res;
+
+ res = __COMPAT_read_enum(type, name, val);
+ SCX_BUG_ON(!res, "enum not found(%s)", name);
+}
+
+#define SCX_ENUM_SET(skel, type, name) do { \
+ __ENUM_set(&skel->rodata->__##name, #type, #name); \
+ } while (0)
+
+
+#include "enums.autogen.h"
+
+#endif /* __SCX_ENUMS_H */
#ifndef __USER_EXIT_INFO_H
#define __USER_EXIT_INFO_H
+#ifdef LSP
+#define __bpf__
+#include "../vmlinux.h"
+#endif
+
enum uei_sizes {
UEI_REASON_LEN = 128,
UEI_MSG_LEN = 1024,
#ifdef __bpf__
-#ifdef LSP
-#include "../vmlinux/vmlinux.h"
-#else
+#ifndef LSP
#include "vmlinux.h"
#endif
#include <bpf/bpf_core_read.h>
const volatile s32 central_cpu;
const volatile u32 nr_cpu_ids = 1; /* !0 for veristat, set during init */
-const volatile u64 slice_ns = SCX_SLICE_DFL;
+const volatile u64 slice_ns;
bool timer_pinned = true;
u64 nr_total, nr_locals, nr_queued, nr_lost_pids;
skel->rodata->central_cpu = 0;
skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus();
+ skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
while ((opt = getopt(argc, argv, "s:c:pvh")) != -1) {
switch (opt) {
char _license[] SEC("license") = "GPL";
const volatile u32 nr_cpus = 32; /* !0 for veristat, set during init */
-const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL;
+const volatile u64 cgrp_slice_ns;
const volatile bool fifo_sched;
u64 cvtime_now;
skel = SCX_OPS_OPEN(flatcg_ops, scx_flatcg);
skel->rodata->nr_cpus = libbpf_num_possible_cpus();
+ skel->rodata->cgrp_slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
while ((opt = getopt(argc, argv, "s:i:dfvh")) != -1) {
double v;
char _license[] SEC("license") = "GPL";
-const volatile u64 slice_ns = SCX_SLICE_DFL;
+const volatile u64 slice_ns;
const volatile u32 stall_user_nth;
const volatile u32 stall_kernel_nth;
const volatile u32 dsp_inf_loop_after;
skel = SCX_OPS_OPEN(qmap_ops, scx_qmap);
+ skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
+
while ((opt = getopt(argc, argv, "s:e:t:T:l:b:PHd:D:Spvh")) != -1) {
switch (opt) {
case 's':
projects / thirdparty / linux.git / commitdiff
