ftrace: make extra rcu_is_watching() validation check optional

Introduce CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING config option to
control whether ftrace low-level code performs additional
rcu_is_watching()-based validation logic in an attempt to catch noinstr
violations.

This check is expected to never be true and is mostly useful for
low-level validation of ftrace subsystem invariants. For most users it
should probably be kept disabled to eliminate unnecessary runtime
overhead.

This improves BPF multi-kretprobe (relying on ftrace and rethook
infrastructure) runtime throughput by 2%, according to BPF benchmarks ([0]).

  [0] https://lore.kernel.org/bpf/CAEf4BzauQ2WKMjZdc9s0rBWa01BYbgwHN6aNDXQSHYia47pQ-w@mail.gmail.com/

Link: https://lore.kernel.org/all/20240418190909.704286-1-andrii@kernel.org/
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Paul E. McKenney <paulmck@kernel.org>
Acked-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>

diff --git a/include/linux/trace_recursion.h b/include/linux/trace_recursion.h
index d48cd92d2364af67cbc6f4da40a1a1913db4843f..24ea8ac049b40877e71094b43b839e15a1b8d212 100644 (file)
--- a/include/linux/trace_recursion.h
+++ b/include/linux/trace_recursion.h
@@ -135,7 +135,7 @@ extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
 # define do_ftrace_record_recursion(ip, pip)   do { } while (0)
 #endif
 
-#ifdef CONFIG_ARCH_WANTS_NO_INSTR
+#ifdef CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING
 # define trace_warn_on_no_rcu(ip)                                      \
        ({                                                              \
                bool __ret = !rcu_is_watching();                        \

diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 47345bf1d4a9f7e850db213999c62ecb02f8fea0..d093e16d01c17b6dede9350003844d488fc26386 100644 (file)
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -974,6 +974,19 @@ config FTRACE_RECORD_RECURSION_SIZE
          This file can be reset, but the limit can not change in
          size at runtime.
 
+config FTRACE_VALIDATE_RCU_IS_WATCHING
+       bool "Validate RCU is on during ftrace execution"
+       depends on FUNCTION_TRACER
+       depends on ARCH_WANTS_NO_INSTR
+       help
+         All callbacks that attach to the function tracing have some sort of
+         protection against recursion. This option is only to verify that
+         ftrace (and other users of ftrace_test_recursion_trylock()) are not
+         called outside of RCU, as if they are, it can cause a race. But it
+         also has a noticeable overhead when enabled.
+
+         If unsure, say N
+
 config RING_BUFFER_RECORD_RECURSION
        bool "Record functions that recurse in the ring buffer"
        depends on FTRACE_RECORD_RECURSION