From 6fa5e3a87cd7838453be66c3a69c2236a1680504 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Tue, 18 Feb 2025 15:58:52 +0100 Subject: [PATCH 01/19] rtla/timerlat: Unify params struct Instead of having separate structs timerlat_top_params and timerlat_hist_params, use one struct timerlat_params for both. This allows code using the structs to be shared between timerlat-top and timerlat-hist. Cc: John Kacur Cc: Luis Goncalves Cc: Gabriele Monaco Cc: Clark Williams Link: https://lore.kernel.org/20250218145859.27762-2-tglozar@redhat.com Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/osnoise.h | 2 + tools/tracing/rtla/src/timerlat.h | 54 ++++++++++++++++++++++ tools/tracing/rtla/src/timerlat_aa.c | 2 - tools/tracing/rtla/src/timerlat_hist.c | 62 +++++--------------------- tools/tracing/rtla/src/timerlat_top.c | 57 +++++------------------ 5 files changed, 78 insertions(+), 99 deletions(-) diff --git a/tools/tracing/rtla/src/osnoise.h b/tools/tracing/rtla/src/osnoise.h index 91835a7d8c2b2..056c8b113dee6 100644 --- a/tools/tracing/rtla/src/osnoise.h +++ b/tools/tracing/rtla/src/osnoise.h @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#pragma once + #include "trace.h" /* diff --git a/tools/tracing/rtla/src/timerlat.h b/tools/tracing/rtla/src/timerlat.h index 88561bfd14f3a..e452d385cb0f4 100644 --- a/tools/tracing/rtla/src/timerlat.h +++ b/tools/tracing/rtla/src/timerlat.h @@ -1,4 +1,58 @@ // SPDX-License-Identifier: GPL-2.0 +#include "utils.h" +#include "osnoise.h" + +struct timerlat_params { + /* Common params */ + char *cpus; + cpu_set_t monitored_cpus; + char *trace_output; + char *cgroup_name; + unsigned long long runtime; + long long stop_us; + long long stop_total_us; + long long timerlat_period_us; + long long print_stack; + int sleep_time; + int output_divisor; + int duration; + int quiet; + int set_sched; + int dma_latency; + int no_aa; + int aa_only; + int dump_tasks; + int cgroup; + int hk_cpus; + int user_workload; + int kernel_workload; + int pretty_output; + int warmup; + int buffer_size; + int deepest_idle_state; + cpu_set_t hk_cpu_set; + struct sched_attr sched_param; + struct trace_events *events; + union { + struct { + /* top only */ + int user_top; + }; + struct { + /* hist only */ + int user_hist; + char no_irq; + char no_thread; + char no_header; + char no_summary; + char no_index; + char with_zeros; + int bucket_size; + int entries; + }; + }; +}; + int timerlat_hist_main(int argc, char *argv[]); int timerlat_top_main(int argc, char *argv[]); int timerlat_main(int argc, char *argv[]); diff --git a/tools/tracing/rtla/src/timerlat_aa.c b/tools/tracing/rtla/src/timerlat_aa.c index 7bd80ee2a5b48..31e66ea2b144c 100644 --- a/tools/tracing/rtla/src/timerlat_aa.c +++ b/tools/tracing/rtla/src/timerlat_aa.c @@ -5,8 +5,6 @@ #include #include -#include "utils.h" -#include "osnoise.h" #include "timerlat.h" #include diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index 6d7d0a2d45b47..fe683a20b0e20 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -14,50 +14,10 @@ #include #include -#include "utils.h" -#include "osnoise.h" #include "timerlat.h" #include "timerlat_aa.h" #include "timerlat_u.h" -struct timerlat_hist_params { - char *cpus; - cpu_set_t monitored_cpus; - char *trace_output; - char *cgroup_name; - unsigned long long runtime; - long long stop_us; - long long stop_total_us; - long long timerlat_period_us; - long long print_stack; - int sleep_time; - int output_divisor; - int duration; - int set_sched; - int dma_latency; - int cgroup; - int hk_cpus; - int no_aa; - int dump_tasks; - int user_workload; - int kernel_workload; - int user_hist; - cpu_set_t hk_cpu_set; - struct sched_attr sched_param; - struct trace_events *events; - char no_irq; - char no_thread; - char no_header; - char no_summary; - char no_index; - char with_zeros; - int bucket_size; - int entries; - int warmup; - int buffer_size; - int deepest_idle_state; -}; - struct timerlat_hist_cpu { int *irq; int *thread; @@ -174,7 +134,7 @@ timerlat_hist_update(struct osnoise_tool *tool, int cpu, unsigned long long context, unsigned long long latency) { - struct timerlat_hist_params *params = tool->params; + struct timerlat_params *params = tool->params; struct timerlat_hist_data *data = tool->data; int entries = data->entries; int bucket; @@ -238,7 +198,7 @@ timerlat_hist_handler(struct trace_seq *s, struct tep_record *record, */ static void timerlat_hist_header(struct osnoise_tool *tool) { - struct timerlat_hist_params *params = tool->params; + struct timerlat_params *params = tool->params; struct timerlat_hist_data *data = tool->data; struct trace_seq *s = tool->trace.seq; char duration[26]; @@ -300,7 +260,7 @@ static void format_summary_value(struct trace_seq *seq, * timerlat_print_summary - print the summary of the hist data to the output */ static void -timerlat_print_summary(struct timerlat_hist_params *params, +timerlat_print_summary(struct timerlat_params *params, struct trace_instance *trace, struct timerlat_hist_data *data) { @@ -427,7 +387,7 @@ timerlat_print_summary(struct timerlat_hist_params *params, } static void -timerlat_print_stats_all(struct timerlat_hist_params *params, +timerlat_print_stats_all(struct timerlat_params *params, struct trace_instance *trace, struct timerlat_hist_data *data) { @@ -575,7 +535,7 @@ timerlat_print_stats_all(struct timerlat_hist_params *params, * timerlat_print_stats - print data for each CPUs */ static void -timerlat_print_stats(struct timerlat_hist_params *params, struct osnoise_tool *tool) +timerlat_print_stats(struct timerlat_params *params, struct osnoise_tool *tool) { struct timerlat_hist_data *data = tool->data; struct trace_instance *trace = &tool->trace; @@ -734,10 +694,10 @@ static void timerlat_hist_usage(char *usage) /* * timerlat_hist_parse_args - allocs, parse and fill the cmd line parameters */ -static struct timerlat_hist_params +static struct timerlat_params *timerlat_hist_parse_args(int argc, char *argv[]) { - struct timerlat_hist_params *params; + struct timerlat_params *params; struct trace_events *tevent; int auto_thresh; int retval; @@ -1017,7 +977,7 @@ static struct timerlat_hist_params * timerlat_hist_apply_config - apply the hist configs to the initialized tool */ static int -timerlat_hist_apply_config(struct osnoise_tool *tool, struct timerlat_hist_params *params) +timerlat_hist_apply_config(struct osnoise_tool *tool, struct timerlat_params *params) { int retval, i; @@ -1122,7 +1082,7 @@ timerlat_hist_apply_config(struct osnoise_tool *tool, struct timerlat_hist_param * timerlat_init_hist - initialize a timerlat hist tool with parameters */ static struct osnoise_tool -*timerlat_init_hist(struct timerlat_hist_params *params) +*timerlat_init_hist(struct timerlat_params *params) { struct osnoise_tool *tool; int nr_cpus; @@ -1170,7 +1130,7 @@ static void stop_hist(int sig) * timerlat_hist_set_signals - handles the signal to stop the tool */ static void -timerlat_hist_set_signals(struct timerlat_hist_params *params) +timerlat_hist_set_signals(struct timerlat_params *params) { signal(SIGINT, stop_hist); if (params->duration) { @@ -1181,7 +1141,7 @@ timerlat_hist_set_signals(struct timerlat_hist_params *params) int timerlat_hist_main(int argc, char *argv[]) { - struct timerlat_hist_params *params; + struct timerlat_params *params; struct osnoise_tool *record = NULL; struct timerlat_u_params params_u; struct osnoise_tool *tool = NULL; diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 05a9403b01d26..97eead91deccd 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -15,45 +15,10 @@ #include #include -#include "utils.h" -#include "osnoise.h" #include "timerlat.h" #include "timerlat_aa.h" #include "timerlat_u.h" -struct timerlat_top_params { - char *cpus; - cpu_set_t monitored_cpus; - char *trace_output; - char *cgroup_name; - unsigned long long runtime; - long long stop_us; - long long stop_total_us; - long long timerlat_period_us; - long long print_stack; - int sleep_time; - int output_divisor; - int duration; - int quiet; - int set_sched; - int dma_latency; - int no_aa; - int aa_only; - int dump_tasks; - int cgroup; - int hk_cpus; - int user_top; - int user_workload; - int kernel_workload; - int pretty_output; - int warmup; - int buffer_size; - int deepest_idle_state; - cpu_set_t hk_cpu_set; - struct sched_attr sched_param; - struct trace_events *events; -}; - struct timerlat_top_cpu { unsigned long long irq_count; unsigned long long thread_count; @@ -194,7 +159,7 @@ timerlat_top_handler(struct trace_seq *s, struct tep_record *record, struct tep_event *event, void *context) { struct trace_instance *trace = context; - struct timerlat_top_params *params; + struct timerlat_params *params; unsigned long long latency, thread; struct osnoise_tool *top; int cpu = record->cpu; @@ -215,7 +180,7 @@ timerlat_top_handler(struct trace_seq *s, struct tep_record *record, /* * timerlat_top_header - print the header of the tool output */ -static void timerlat_top_header(struct timerlat_top_params *params, struct osnoise_tool *top) +static void timerlat_top_header(struct timerlat_params *params, struct osnoise_tool *top) { struct trace_seq *s = top->trace.seq; char duration[26]; @@ -263,7 +228,7 @@ static const char *no_value = " -"; static void timerlat_top_print(struct osnoise_tool *top, int cpu) { - struct timerlat_top_params *params = top->params; + struct timerlat_params *params = top->params; struct timerlat_top_data *data = top->data; struct timerlat_top_cpu *cpu_data = &data->cpu_data[cpu]; int divisor = params->output_divisor; @@ -327,7 +292,7 @@ static void timerlat_top_print_sum(struct osnoise_tool *top, struct timerlat_top_cpu *summary) { const char *split = "----------------------------------------"; - struct timerlat_top_params *params = top->params; + struct timerlat_params *params = top->params; unsigned long long count = summary->irq_count; int divisor = params->output_divisor; struct trace_seq *s = top->trace.seq; @@ -404,7 +369,7 @@ static void clear_terminal(struct trace_seq *seq) * timerlat_print_stats - print data for all cpus */ static void -timerlat_print_stats(struct timerlat_top_params *params, struct osnoise_tool *top) +timerlat_print_stats(struct timerlat_params *params, struct osnoise_tool *top) { struct trace_instance *trace = &top->trace; struct timerlat_top_cpu summary; @@ -505,10 +470,10 @@ static void timerlat_top_usage(char *usage) /* * timerlat_top_parse_args - allocs, parse and fill the cmd line parameters */ -static struct timerlat_top_params +static struct timerlat_params *timerlat_top_parse_args(int argc, char **argv) { - struct timerlat_top_params *params; + struct timerlat_params *params; struct trace_events *tevent; long long auto_thresh; int retval; @@ -765,7 +730,7 @@ static struct timerlat_top_params * timerlat_top_apply_config - apply the top configs to the initialized tool */ static int -timerlat_top_apply_config(struct osnoise_tool *top, struct timerlat_top_params *params) +timerlat_top_apply_config(struct osnoise_tool *top, struct timerlat_params *params) { int retval; int i; @@ -876,7 +841,7 @@ timerlat_top_apply_config(struct osnoise_tool *top, struct timerlat_top_params * * timerlat_init_top - initialize a timerlat top tool with parameters */ static struct osnoise_tool -*timerlat_init_top(struct timerlat_top_params *params) +*timerlat_init_top(struct timerlat_params *params) { struct osnoise_tool *top; int nr_cpus; @@ -924,7 +889,7 @@ static void stop_top(int sig) * timerlat_top_set_signals - handles the signal to stop the tool */ static void -timerlat_top_set_signals(struct timerlat_top_params *params) +timerlat_top_set_signals(struct timerlat_params *params) { signal(SIGINT, stop_top); if (params->duration) { @@ -935,7 +900,7 @@ timerlat_top_set_signals(struct timerlat_top_params *params) int timerlat_top_main(int argc, char *argv[]) { - struct timerlat_top_params *params; + struct timerlat_params *params; struct osnoise_tool *record = NULL; struct timerlat_u_params params_u; struct osnoise_tool *top = NULL; From 8a635c3856ddb74ed3fe7c856b271cdfeb65f293 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Tue, 18 Feb 2025 15:58:53 +0100 Subject: [PATCH 02/19] tools/build: Add bpftool-skeletons feature test Add bpftool-skeletons feature test, testing the presence of a bpftool capable of generating skeletons. This is to be used for tools that do not require building their own bootstrap bpftool from the kernel source tree. Cc: John Kacur Cc: Luis Goncalves Cc: Gabriele Monaco Cc: Clark Williams Link: https://lore.kernel.org/20250218145859.27762-3-tglozar@redhat.com Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/build/Makefile.feature | 3 ++- tools/build/feature/Makefile | 3 +++ tools/scripts/Makefile.include | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index 1931b63213146..1f44ca677ad3d 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -135,7 +135,8 @@ FEATURE_TESTS_EXTRA := \ libbpf-bpf_create_map \ libpfm4 \ libdebuginfod \ - clang-bpf-co-re + clang-bpf-co-re \ + bpftool-skeletons FEATURE_TESTS ?= $(FEATURE_TESTS_BASIC) diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index cb1e3e2feedf3..4f9c1d950f5db 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -418,6 +418,9 @@ $(OUTPUT)test-file-handle.bin: $(OUTPUT)test-libpfm4.bin: $(BUILD) -lpfm +$(OUTPUT)test-bpftool-skeletons.bin: + $(BPFTOOL) version | grep '^features:.*skeletons' \ + > $(@:.bin=.make.output) 2>&1 ############################### clean: diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include index 0aa4005017c72..71bbe52721b35 100644 --- a/tools/scripts/Makefile.include +++ b/tools/scripts/Makefile.include @@ -91,6 +91,9 @@ LLVM_CONFIG ?= llvm-config LLVM_OBJCOPY ?= llvm-objcopy LLVM_STRIP ?= llvm-strip +# Some tools require bpftool +BPFTOOL ?= bpftool + ifeq ($(CC_NO_CLANG), 1) EXTRA_WARNINGS += -Wstrict-aliasing=3 From 9dc3766ed07c95c9a77fa98dcbc83dcb7f49df3d Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Tue, 18 Feb 2025 15:58:54 +0100 Subject: [PATCH 03/19] rtla: Add optional dependency on BPF tooling If tooling required for building BPF CO-RE skeletons is present (that is, libbpf, clang with BPF CO-RE support, and bpftool), turn on HAVE_BPF_SKEL flag. Those requirements are similar to what perf requires, with the difference of using system libbpf and bpftool instead of in-tree versions. rtla can be forcefully built without BPF skeleton support by setting BUILD_BPF_SKEL=0 manually; in that case, a warning is displayed. Cc: John Kacur Cc: Luis Goncalves Cc: Gabriele Monaco Cc: Clark Williams Link: https://lore.kernel.org/20250218145859.27762-4-tglozar@redhat.com Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/Makefile | 6 +++++ tools/tracing/rtla/Makefile.config | 42 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/tools/tracing/rtla/Makefile b/tools/tracing/rtla/Makefile index 8b5101457c70b..0aa116f7b355d 100644 --- a/tools/tracing/rtla/Makefile +++ b/tools/tracing/rtla/Makefile @@ -33,9 +33,15 @@ DOCSRC := ../../../Documentation/tools/rtla/ FEATURE_TESTS := libtraceevent FEATURE_TESTS += libtracefs FEATURE_TESTS += libcpupower +FEATURE_TESTS += libbpf +FEATURE_TESTS += clang-bpf-co-re +FEATURE_TESTS += bpftool-skeletons FEATURE_DISPLAY := libtraceevent FEATURE_DISPLAY += libtracefs FEATURE_DISPLAY += libcpupower +FEATURE_DISPLAY += libbpf +FEATURE_DISPLAY += clang-bpf-co-re +FEATURE_DISPLAY += bpftool-skeletons ifeq ($(V),1) Q = diff --git a/tools/tracing/rtla/Makefile.config b/tools/tracing/rtla/Makefile.config index 92a6e12e42d3a..5f2231d8d6266 100644 --- a/tools/tracing/rtla/Makefile.config +++ b/tools/tracing/rtla/Makefile.config @@ -53,6 +53,48 @@ else $(info Please install libcpupower-dev/kernel-tools-libs-devel) endif +ifndef BUILD_BPF_SKEL + # BPF skeletons are used to implement improved sample collection, enable + # them by default. + BUILD_BPF_SKEL := 1 +endif + +ifeq ($(BUILD_BPF_SKEL),0) + $(info BPF skeleton support disabled, building without BPF skeleton support.) +endif + +$(call feature_check,libbpf) +ifeq ($(feature-libbpf), 1) + $(call detected,CONFIG_LIBBPF) +else + $(info libbpf is missing, building without BPF skeleton support.) + $(info Please install libbpf-dev/libbpf-devel) + BUILD_BPF_SKEL := 0 +endif + +$(call feature_check,clang-bpf-co-re) +ifeq ($(feature-clang-bpf-co-re), 1) + $(call detected,CONFIG_CLANG_BPF_CO_RE) +else + $(info clang is missing or does not support BPF CO-RE, building without BPF skeleton support.) + $(info Please install clang) + BUILD_BPF_SKEL := 0 +endif + +$(call feature_check,bpftool-skeletons) +ifeq ($(feature-bpftool-skeletons), 1) + $(call detected,CONFIG_BPFTOOL_SKELETONS) +else + $(info bpftool is missing or not supporting skeletons, building without BPF skeleton support.) + $(info Please install bpftool) + BUILD_BPF_SKEL := 0 +endif + +ifeq ($(BUILD_BPF_SKEL),1) + CFLAGS += -DHAVE_BPF_SKEL + EXTLIBS += -lbpf +endif + ifeq ($(STOP_ERROR),1) $(error Please, check the errors above.) endif From e34293ddcebd6bd39ff04666f010fce8d1f5e15a Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Tue, 18 Feb 2025 15:58:55 +0100 Subject: [PATCH 04/19] rtla/timerlat: Add BPF skeleton to collect samples Add BPF program that attaches to the osnoise:timerlat_sample tracepoint and collects both the summary and the histogram (if requested) into BPF maps (one map of each kind per context). The program is designed to be used for both timerlat-top and timerlat-hist. If using with timerlat-top, the "entries" parameter is set to zero, which prevents the BPF program from recording histogram entries. In that case, the maps for histograms do not have to be created, as the BPF verifier will identify the code using them as unreachable. An IRQ or thread latency threshold might be supplied to stop recording if hit, similar to the timerlat tracer threshold, which stops ftrace tracing if hit. A BPF ringbuffer is used to signal threshold overflow to userspace. In aa-only mode, this is the only function of the BPF program. Cc: John Kacur Cc: Luis Goncalves Cc: Gabriele Monaco Cc: Clark Williams Link: https://lore.kernel.org/20250218145859.27762-5-tglozar@redhat.com Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/.gitignore | 1 + tools/tracing/rtla/Makefile | 14 ++- tools/tracing/rtla/src/Build | 1 + tools/tracing/rtla/src/timerlat.bpf.c | 149 +++++++++++++++++++++++ tools/tracing/rtla/src/timerlat_bpf.c | 166 ++++++++++++++++++++++++++ tools/tracing/rtla/src/timerlat_bpf.h | 59 +++++++++ 6 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 tools/tracing/rtla/src/timerlat.bpf.c create mode 100644 tools/tracing/rtla/src/timerlat_bpf.c create mode 100644 tools/tracing/rtla/src/timerlat_bpf.h diff --git a/tools/tracing/rtla/.gitignore b/tools/tracing/rtla/.gitignore index 293f0dbb0ca2e..1a394ad26cc1e 100644 --- a/tools/tracing/rtla/.gitignore +++ b/tools/tracing/rtla/.gitignore @@ -4,3 +4,4 @@ rtla-static fixdep feature FEATURE-DUMP +*.skel.h diff --git a/tools/tracing/rtla/Makefile b/tools/tracing/rtla/Makefile index 0aa116f7b355d..557af322be610 100644 --- a/tools/tracing/rtla/Makefile +++ b/tools/tracing/rtla/Makefile @@ -73,6 +73,17 @@ CFLAGS += $(INCLUDES) $(LIB_INCLUDES) export CFLAGS OUTPUT srctree +ifeq ($(BUILD_BPF_SKEL),1) +src/timerlat.bpf.o: src/timerlat.bpf.c + $(QUIET_CLANG)$(CLANG) -g -O2 -target bpf -c $(filter %.c,$^) -o $@ + +src/timerlat.skel.h: src/timerlat.bpf.o + $(QUIET_GENSKEL)$(BPFTOOL) gen skeleton $< > $@ +else +src/timerlat.skel.h: + $(Q)echo '/* BPF skeleton is disabled */' > src/timerlat.skel.h +endif + $(RTLA): $(RTLA_IN) $(QUIET_LINK)$(CC) $(LDFLAGS) -o $(RTLA) $(RTLA_IN) $(EXTLIBS) @@ -83,7 +94,7 @@ static: $(RTLA_IN) rtla.%: fixdep FORCE make -f $(srctree)/tools/build/Makefile.build dir=. $@ -$(RTLA_IN): fixdep FORCE +$(RTLA_IN): fixdep FORCE src/timerlat.skel.h make $(build)=rtla clean: doc_clean fixdep-clean @@ -91,6 +102,7 @@ clean: doc_clean fixdep-clean $(Q)find . -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete $(Q)rm -f rtla rtla-static fixdep FEATURE-DUMP rtla-* $(Q)rm -rf feature + $(Q)rm -f src/timerlat.bpf.o src/timerlat.skel.h check: $(RTLA) RTLA=$(RTLA) prove -o -f tests/ .PHONY: FORCE clean check diff --git a/tools/tracing/rtla/src/Build b/tools/tracing/rtla/src/Build index dbed9e31829b9..7bb7e39e391a2 100644 --- a/tools/tracing/rtla/src/Build +++ b/tools/tracing/rtla/src/Build @@ -8,4 +8,5 @@ rtla-y += timerlat_top.o rtla-y += timerlat_hist.o rtla-y += timerlat_u.o rtla-y += timerlat_aa.o +rtla-y += timerlat_bpf.o rtla-y += rtla.o diff --git a/tools/tracing/rtla/src/timerlat.bpf.c b/tools/tracing/rtla/src/timerlat.bpf.c new file mode 100644 index 0000000000000..96196d46e1701 --- /dev/null +++ b/tools/tracing/rtla/src/timerlat.bpf.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "timerlat_bpf.h" + +#define nosubprog __always_inline +#define MAX_ENTRIES_DEFAULT 4096 + +char LICENSE[] SEC("license") = "GPL"; + +struct trace_event_raw_timerlat_sample { + unsigned long long timer_latency; + int context; +} __attribute__((preserve_access_index)); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, MAX_ENTRIES_DEFAULT); + __type(key, unsigned int); + __type(value, unsigned long long); +} hist_irq SEC(".maps"), hist_thread SEC(".maps"), hist_user SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, SUMMARY_FIELD_N); + __type(key, unsigned int); + __type(value, unsigned long long); +} summary_irq SEC(".maps"), summary_thread SEC(".maps"), summary_user SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1); +} signal_stop_tracing SEC(".maps"); + +/* Params to be set by rtla */ +const volatile int bucket_size = 1; +const volatile int output_divisor = 1000; +const volatile int entries = 256; +const volatile int irq_threshold; +const volatile int thread_threshold; +const volatile bool aa_only; + +int stop_tracing; + +nosubprog unsigned long long map_get(void *map, + unsigned int key) +{ + unsigned long long *value_ptr; + + value_ptr = bpf_map_lookup_elem(map, &key); + + return !value_ptr ? 0 : *value_ptr; +} + +nosubprog void map_set(void *map, + unsigned int key, + unsigned long long value) +{ + bpf_map_update_elem(map, &key, &value, BPF_ANY); +} + +nosubprog void map_increment(void *map, + unsigned int key) +{ + map_set(map, key, map_get(map, key) + 1); +} + +nosubprog void update_main_hist(void *map, + int bucket) +{ + if (entries == 0) + /* No histogram */ + return; + + if (bucket >= entries) + /* Overflow */ + return; + + map_increment(map, bucket); +} + +nosubprog void update_summary(void *map, + unsigned long long latency, + int bucket) +{ + if (aa_only) + /* Auto-analysis only, nothing to be done here */ + return; + + map_set(map, SUMMARY_CURRENT, latency); + + if (bucket >= entries) + /* Overflow */ + map_increment(map, SUMMARY_OVERFLOW); + + if (latency > map_get(map, SUMMARY_MAX)) + map_set(map, SUMMARY_MAX, latency); + + if (latency < map_get(map, SUMMARY_MIN) || map_get(map, SUMMARY_COUNT) == 0) + map_set(map, SUMMARY_MIN, latency); + + map_increment(map, SUMMARY_COUNT); + map_set(map, SUMMARY_SUM, map_get(map, SUMMARY_SUM) + latency); +} + +nosubprog void set_stop_tracing(void) +{ + int value = 0; + + /* Suppress further sample processing */ + stop_tracing = 1; + + /* Signal to userspace */ + bpf_ringbuf_output(&signal_stop_tracing, &value, sizeof(value), 0); +} + +SEC("tp/osnoise/timerlat_sample") +int handle_timerlat_sample(struct trace_event_raw_timerlat_sample *tp_args) +{ + unsigned long long latency, latency_us; + int bucket; + + if (stop_tracing) + return 0; + + latency = tp_args->timer_latency / output_divisor; + latency_us = tp_args->timer_latency / 1000; + bucket = latency / bucket_size; + + if (tp_args->context == 0) { + update_main_hist(&hist_irq, bucket); + update_summary(&summary_irq, latency, bucket); + + if (irq_threshold != 0 && latency_us >= irq_threshold) + set_stop_tracing(); + } else if (tp_args->context == 1) { + update_main_hist(&hist_thread, bucket); + update_summary(&summary_thread, latency, bucket); + + if (thread_threshold != 0 && latency_us >= thread_threshold) + set_stop_tracing(); + } else { + update_main_hist(&hist_user, bucket); + update_summary(&summary_user, latency, bucket); + } + + return 0; +} diff --git a/tools/tracing/rtla/src/timerlat_bpf.c b/tools/tracing/rtla/src/timerlat_bpf.c new file mode 100644 index 0000000000000..5abee884037ae --- /dev/null +++ b/tools/tracing/rtla/src/timerlat_bpf.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifdef HAVE_BPF_SKEL +#include "timerlat.h" +#include "timerlat_bpf.h" +#include "timerlat.skel.h" + +static struct timerlat_bpf *bpf; + +/* + * timerlat_bpf_init - load and initialize BPF program to collect timerlat data + */ +int timerlat_bpf_init(struct timerlat_params *params) +{ + int err; + + debug_msg("Loading BPF program\n"); + + bpf = timerlat_bpf__open(); + if (!bpf) + return 1; + + /* Pass common options */ + bpf->rodata->output_divisor = params->output_divisor; + bpf->rodata->entries = params->entries; + bpf->rodata->irq_threshold = params->stop_us; + bpf->rodata->thread_threshold = params->stop_total_us; + bpf->rodata->aa_only = params->aa_only; + + if (params->entries != 0) { + /* Pass histogram options */ + bpf->rodata->bucket_size = params->bucket_size; + + /* Set histogram array sizes */ + bpf_map__set_max_entries(bpf->maps.hist_irq, params->entries); + bpf_map__set_max_entries(bpf->maps.hist_thread, params->entries); + bpf_map__set_max_entries(bpf->maps.hist_user, params->entries); + } else { + /* No entries, disable histogram */ + bpf_map__set_autocreate(bpf->maps.hist_irq, false); + bpf_map__set_autocreate(bpf->maps.hist_thread, false); + bpf_map__set_autocreate(bpf->maps.hist_user, false); + } + + if (params->aa_only) { + /* Auto-analysis only, disable summary */ + bpf_map__set_autocreate(bpf->maps.summary_irq, false); + bpf_map__set_autocreate(bpf->maps.summary_thread, false); + bpf_map__set_autocreate(bpf->maps.summary_user, false); + } + + /* Load and verify BPF program */ + err = timerlat_bpf__load(bpf); + if (err) { + timerlat_bpf__destroy(bpf); + return err; + } + + return 0; +} + +/* + * timerlat_bpf_attach - attach BPF program to collect timerlat data + */ +int timerlat_bpf_attach(void) +{ + debug_msg("Attaching BPF program\n"); + + return timerlat_bpf__attach(bpf); +} + +/* + * timerlat_bpf_detach - detach BPF program to collect timerlat data + */ +void timerlat_bpf_detach(void) +{ + timerlat_bpf__detach(bpf); +} + +/* + * timerlat_bpf_detach - destroy BPF program to collect timerlat data + */ +void timerlat_bpf_destroy(void) +{ + timerlat_bpf__destroy(bpf); +} + +static int handle_rb_event(void *ctx, void *data, size_t data_sz) +{ + return 0; +} + +/* + * timerlat_bpf_wait - wait until tracing is stopped or signal + */ +int timerlat_bpf_wait(int timeout) +{ + struct ring_buffer *rb; + int retval; + + rb = ring_buffer__new(bpf_map__fd(bpf->maps.signal_stop_tracing), + handle_rb_event, NULL, NULL); + retval = ring_buffer__poll(rb, timeout * 1000); + ring_buffer__free(rb); + + return retval; +} + +static int get_value(struct bpf_map *map_irq, + struct bpf_map *map_thread, + struct bpf_map *map_user, + int key, + long long *value_irq, + long long *value_thread, + long long *value_user, + int cpus) +{ + int err; + + err = bpf_map__lookup_elem(map_irq, &key, + sizeof(unsigned int), value_irq, + sizeof(long long) * cpus, 0); + if (err) + return err; + err = bpf_map__lookup_elem(map_thread, &key, + sizeof(unsigned int), value_thread, + sizeof(long long) * cpus, 0); + if (err) + return err; + err = bpf_map__lookup_elem(map_user, &key, + sizeof(unsigned int), value_user, + sizeof(long long) * cpus, 0); + if (err) + return err; + return 0; +} + +/* + * timerlat_bpf_get_hist_value - get value from BPF hist map + */ +int timerlat_bpf_get_hist_value(int key, + long long *value_irq, + long long *value_thread, + long long *value_user, + int cpus) +{ + return get_value(bpf->maps.hist_irq, + bpf->maps.hist_thread, + bpf->maps.hist_user, + key, value_irq, value_thread, value_user, cpus); +} + +/* + * timerlat_bpf_get_summary_value - get value from BPF summary map + */ +int timerlat_bpf_get_summary_value(enum summary_field key, + long long *value_irq, + long long *value_thread, + long long *value_user, + int cpus) +{ + return get_value(bpf->maps.summary_irq, + bpf->maps.summary_thread, + bpf->maps.summary_user, + key, value_irq, value_thread, value_user, cpus); +} +#endif /* HAVE_BPF_SKEL */ diff --git a/tools/tracing/rtla/src/timerlat_bpf.h b/tools/tracing/rtla/src/timerlat_bpf.h new file mode 100644 index 0000000000000..f1b54dbddb0e2 --- /dev/null +++ b/tools/tracing/rtla/src/timerlat_bpf.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#pragma once + +enum summary_field { + SUMMARY_CURRENT, + SUMMARY_MIN, + SUMMARY_MAX, + SUMMARY_COUNT, + SUMMARY_SUM, + SUMMARY_OVERFLOW, + SUMMARY_FIELD_N +}; + +#ifndef __bpf__ +#ifdef HAVE_BPF_SKEL +int timerlat_bpf_init(struct timerlat_params *params); +int timerlat_bpf_attach(void); +void timerlat_bpf_detach(void); +void timerlat_bpf_destroy(void); +int timerlat_bpf_wait(int timeout); +int timerlat_bpf_get_hist_value(int key, + long long *value_irq, + long long *value_thread, + long long *value_user, + int cpus); +int timerlat_bpf_get_summary_value(enum summary_field key, + long long *value_irq, + long long *value_thread, + long long *value_user, + int cpus); +static inline int have_libbpf_support(void) { return 1; } +#else +static inline int timerlat_bpf_init(struct timerlat_params *params) +{ + return -1; +} +static inline int timerlat_bpf_attach(void) { return -1; } +static inline void timerlat_bpf_detach(void) { }; +static inline void timerlat_bpf_destroy(void) { }; +static inline int timerlat_bpf_wait(int timeout) { return -1; } +static inline int timerlat_bpf_get_hist_value(int key, + long long *value_irq, + long long *value_thread, + long long *value_user, + int cpus) +{ + return -1; +} +static inline int timerlat_bpf_get_summary_value(enum summary_field key, + long long *value_irq, + long long *value_thread, + long long *value_user, + int cpus) +{ + return -1; +} +static inline int have_libbpf_support(void) { return 0; } +#endif /* HAVE_BPF_SKEL */ +#endif /* __bpf__ */ From fd7925cbb729c7744ffbac95928be4598354c540 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Tue, 18 Feb 2025 15:58:56 +0100 Subject: [PATCH 05/19] rtla/timerlat_hist: Use BPF to collect samples Collect samples using BPF program instead of pulling them from tracefs. If the osnoise:timerlat_sample tracepoint is unavailable or the BPF program fails to load for whatever reason, rtla falls back to the old implementation. The collection of samples using the BPF program is fully self-contained and requires no activity of the userspace part of rtla during the measurement. Thus, instead of waking up every second to collect samples, rtla simply sleeps until woken up by a signal or threshold overflow. Cc: John Kacur Cc: Luis Goncalves Cc: Gabriele Monaco Cc: Clark Williams Link: https://lore.kernel.org/20250218145859.27762-6-tglozar@redhat.com Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/timerlat_hist.c | 167 +++++++++++++++++++++---- 1 file changed, 146 insertions(+), 21 deletions(-) diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index fe683a20b0e20..202d99a598ba8 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -17,6 +17,7 @@ #include "timerlat.h" #include "timerlat_aa.h" #include "timerlat_u.h" +#include "timerlat_bpf.h" struct timerlat_hist_cpu { int *irq; @@ -193,6 +194,89 @@ timerlat_hist_handler(struct trace_seq *s, struct tep_record *record, return 0; } +/* + * timerlat_hist_bpf_pull_data - copy data from BPF maps into userspace + */ +static int timerlat_hist_bpf_pull_data(struct osnoise_tool *tool) +{ + struct timerlat_hist_data *data = tool->data; + int i, j, err; + long long value_irq[data->nr_cpus], + value_thread[data->nr_cpus], + value_user[data->nr_cpus]; + + /* Pull histogram */ + for (i = 0; i < data->entries; i++) { + err = timerlat_bpf_get_hist_value(i, value_irq, value_thread, + value_user, data->nr_cpus); + if (err) + return err; + for (j = 0; j < data->nr_cpus; j++) { + data->hist[j].irq[i] = value_irq[j]; + data->hist[j].thread[i] = value_thread[j]; + data->hist[j].user[i] = value_user[j]; + } + } + + /* Pull summary */ + err = timerlat_bpf_get_summary_value(SUMMARY_COUNT, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->hist[i].irq_count = value_irq[i]; + data->hist[i].thread_count = value_thread[i]; + data->hist[i].user_count = value_user[i]; + } + + err = timerlat_bpf_get_summary_value(SUMMARY_MIN, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->hist[i].min_irq = value_irq[i]; + data->hist[i].min_thread = value_thread[i]; + data->hist[i].min_user = value_user[i]; + } + + err = timerlat_bpf_get_summary_value(SUMMARY_MAX, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->hist[i].max_irq = value_irq[i]; + data->hist[i].max_thread = value_thread[i]; + data->hist[i].max_user = value_user[i]; + } + + err = timerlat_bpf_get_summary_value(SUMMARY_SUM, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->hist[i].sum_irq = value_irq[i]; + data->hist[i].sum_thread = value_thread[i]; + data->hist[i].sum_user = value_user[i]; + } + + err = timerlat_bpf_get_summary_value(SUMMARY_OVERFLOW, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->hist[i].irq[data->entries] = value_irq[i]; + data->hist[i].thread[data->entries] = value_thread[i]; + data->hist[i].user[data->entries] = value_user[i]; + } + + return 0; +} + /* * timerlat_hist_header - print the header of the tracer to the output */ @@ -1152,6 +1236,7 @@ int timerlat_hist_main(int argc, char *argv[]) pthread_t timerlat_u; int retval; int nr_cpus, i; + bool no_bpf = false; params = timerlat_hist_parse_args(argc, argv); if (!params) @@ -1177,6 +1262,24 @@ int timerlat_hist_main(int argc, char *argv[]) */ hist_inst = trace; + if (getenv("RTLA_NO_BPF") && strncmp(getenv("RTLA_NO_BPF"), "1", 2) == 0) { + debug_msg("RTLA_NO_BPF set, disabling BPF\n"); + no_bpf = true; + } + + if (!no_bpf && !tep_find_event_by_name(trace->tep, "osnoise", "timerlat_sample")) { + debug_msg("osnoise:timerlat_sample missing, disabling BPF\n"); + no_bpf = true; + } + + if (!no_bpf) { + retval = timerlat_bpf_init(params); + if (retval) { + debug_msg("Could not enable BPF\n"); + no_bpf = true; + } + } + retval = enable_timerlat(trace); if (retval) { err_msg("Failed to enable timerlat tracer\n"); @@ -1304,35 +1407,55 @@ int timerlat_hist_main(int argc, char *argv[]) trace_instance_start(&record->trace); if (!params->no_aa) trace_instance_start(&aa->trace); - trace_instance_start(trace); + if (no_bpf) { + trace_instance_start(trace); + } else { + retval = timerlat_bpf_attach(); + if (retval) { + err_msg("Error attaching BPF program\n"); + goto out_hist; + } + } tool->start_time = time(NULL); timerlat_hist_set_signals(params); - while (!stop_tracing) { - sleep(params->sleep_time); - - retval = tracefs_iterate_raw_events(trace->tep, - trace->inst, - NULL, - 0, - collect_registered_events, - trace); - if (retval < 0) { - err_msg("Error iterating on events\n"); - goto out_hist; - } - - if (osnoise_trace_is_off(tool, record)) - break; + if (no_bpf) { + while (!stop_tracing) { + sleep(params->sleep_time); + + retval = tracefs_iterate_raw_events(trace->tep, + trace->inst, + NULL, + 0, + collect_registered_events, + trace); + if (retval < 0) { + err_msg("Error iterating on events\n"); + goto out_hist; + } - /* is there still any user-threads ? */ - if (params->user_workload) { - if (params_u.stopped_running) { - debug_msg("timerlat user-space threads stopped!\n"); + if (osnoise_trace_is_off(tool, record)) break; + + /* is there still any user-threads ? */ + if (params->user_workload) { + if (params_u.stopped_running) { + debug_msg("timerlat user-space threads stopped!\n"); + break; + } } } + } else + timerlat_bpf_wait(-1); + + if (!no_bpf) { + timerlat_bpf_detach(); + retval = timerlat_hist_bpf_pull_data(tool); + if (retval) { + err_msg("Error pulling BPF data\n"); + goto out_hist; + } } if (params->user_workload && !params_u.stopped_running) { @@ -1376,6 +1499,8 @@ int timerlat_hist_main(int argc, char *argv[]) osnoise_destroy_tool(tool); free(params); free_cpu_idle_disable_states(); + if (!no_bpf) + timerlat_bpf_destroy(); out_exit: exit(return_value); } From 18923806b1291102cad3a6b713006c7e7f563534 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Tue, 18 Feb 2025 15:58:57 +0100 Subject: [PATCH 06/19] rtla/timerlat_top: Move divisor to update Unlike timerlat-hist, timerlat-top applies the output divisor used to set ns/us mode when printing results instead of applying it when collecting the samples. Move the application of the divisor from timerlat_top_print into timerlat_top_update to make it consistent with timerlat-hist. Cc: John Kacur Cc: Luis Goncalves Cc: Gabriele Monaco Cc: Clark Williams Link: https://lore.kernel.org/20250218145859.27762-7-tglozar@redhat.com Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/timerlat_top.c | 54 +++++++++++++-------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 97eead91deccd..a3d9e8f67a4f7 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -127,9 +127,13 @@ timerlat_top_update(struct osnoise_tool *tool, int cpu, unsigned long long thread, unsigned long long latency) { + struct timerlat_params *params = tool->params; struct timerlat_top_data *data = tool->data; struct timerlat_top_cpu *cpu_data = &data->cpu_data[cpu]; + if (params->output_divisor) + latency = latency / params->output_divisor; + if (!thread) { cpu_data->irq_count++; cpu_data->cur_irq = latency; @@ -231,12 +235,8 @@ static void timerlat_top_print(struct osnoise_tool *top, int cpu) struct timerlat_params *params = top->params; struct timerlat_top_data *data = top->data; struct timerlat_top_cpu *cpu_data = &data->cpu_data[cpu]; - int divisor = params->output_divisor; struct trace_seq *s = top->trace.seq; - if (divisor == 0) - return; - /* * Skip if no data is available: is this cpu offline? */ @@ -251,20 +251,20 @@ static void timerlat_top_print(struct osnoise_tool *top, int cpu) if (!cpu_data->irq_count) { trace_seq_printf(s, "%s %s %s %s |", no_value, no_value, no_value, no_value); } else { - trace_seq_printf(s, "%9llu ", cpu_data->cur_irq / params->output_divisor); - trace_seq_printf(s, "%9llu ", cpu_data->min_irq / params->output_divisor); - trace_seq_printf(s, "%9llu ", (cpu_data->sum_irq / cpu_data->irq_count) / divisor); - trace_seq_printf(s, "%9llu |", cpu_data->max_irq / divisor); + trace_seq_printf(s, "%9llu ", cpu_data->cur_irq); + trace_seq_printf(s, "%9llu ", cpu_data->min_irq); + trace_seq_printf(s, "%9llu ", cpu_data->sum_irq / cpu_data->irq_count); + trace_seq_printf(s, "%9llu |", cpu_data->max_irq); } if (!cpu_data->thread_count) { trace_seq_printf(s, "%s %s %s %s", no_value, no_value, no_value, no_value); } else { - trace_seq_printf(s, "%9llu ", cpu_data->cur_thread / divisor); - trace_seq_printf(s, "%9llu ", cpu_data->min_thread / divisor); + trace_seq_printf(s, "%9llu ", cpu_data->cur_thread); + trace_seq_printf(s, "%9llu ", cpu_data->min_thread); trace_seq_printf(s, "%9llu ", - (cpu_data->sum_thread / cpu_data->thread_count) / divisor); - trace_seq_printf(s, "%9llu", cpu_data->max_thread / divisor); + cpu_data->sum_thread / cpu_data->thread_count); + trace_seq_printf(s, "%9llu", cpu_data->max_thread); } if (!params->user_top) { @@ -277,11 +277,11 @@ static void timerlat_top_print(struct osnoise_tool *top, int cpu) if (!cpu_data->user_count) { trace_seq_printf(s, "%s %s %s %s\n", no_value, no_value, no_value, no_value); } else { - trace_seq_printf(s, "%9llu ", cpu_data->cur_user / divisor); - trace_seq_printf(s, "%9llu ", cpu_data->min_user / divisor); + trace_seq_printf(s, "%9llu ", cpu_data->cur_user); + trace_seq_printf(s, "%9llu ", cpu_data->min_user); trace_seq_printf(s, "%9llu ", - (cpu_data->sum_user / cpu_data->user_count) / divisor); - trace_seq_printf(s, "%9llu\n", cpu_data->max_user / divisor); + cpu_data->sum_user / cpu_data->user_count); + trace_seq_printf(s, "%9llu\n", cpu_data->max_user); } } @@ -294,13 +294,9 @@ timerlat_top_print_sum(struct osnoise_tool *top, struct timerlat_top_cpu *summar const char *split = "----------------------------------------"; struct timerlat_params *params = top->params; unsigned long long count = summary->irq_count; - int divisor = params->output_divisor; struct trace_seq *s = top->trace.seq; int e = 0; - if (divisor == 0) - return; - /* * Skip if no data is available: is this cpu offline? */ @@ -323,19 +319,19 @@ timerlat_top_print_sum(struct osnoise_tool *top, struct timerlat_top_cpu *summar trace_seq_printf(s, " %s %s %s |", no_value, no_value, no_value); } else { trace_seq_printf(s, " "); - trace_seq_printf(s, "%9llu ", summary->min_irq / params->output_divisor); - trace_seq_printf(s, "%9llu ", (summary->sum_irq / summary->irq_count) / divisor); - trace_seq_printf(s, "%9llu |", summary->max_irq / divisor); + trace_seq_printf(s, "%9llu ", summary->min_irq); + trace_seq_printf(s, "%9llu ", summary->sum_irq / summary->irq_count); + trace_seq_printf(s, "%9llu |", summary->max_irq); } if (!summary->thread_count) { trace_seq_printf(s, "%s %s %s %s", no_value, no_value, no_value, no_value); } else { trace_seq_printf(s, " "); - trace_seq_printf(s, "%9llu ", summary->min_thread / divisor); + trace_seq_printf(s, "%9llu ", summary->min_thread); trace_seq_printf(s, "%9llu ", - (summary->sum_thread / summary->thread_count) / divisor); - trace_seq_printf(s, "%9llu", summary->max_thread / divisor); + summary->sum_thread / summary->thread_count); + trace_seq_printf(s, "%9llu", summary->max_thread); } if (!params->user_top) { @@ -349,10 +345,10 @@ timerlat_top_print_sum(struct osnoise_tool *top, struct timerlat_top_cpu *summar trace_seq_printf(s, " %s %s %s |", no_value, no_value, no_value); } else { trace_seq_printf(s, " "); - trace_seq_printf(s, "%9llu ", summary->min_user / divisor); + trace_seq_printf(s, "%9llu ", summary->min_user); trace_seq_printf(s, "%9llu ", - (summary->sum_user / summary->user_count) / divisor); - trace_seq_printf(s, "%9llu\n", summary->max_user / divisor); + summary->sum_user / summary->user_count); + trace_seq_printf(s, "%9llu\n", summary->max_user); } } From 9a82a3fd9ef2afaafbd6f8059609cdead0f97f15 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Tue, 18 Feb 2025 15:58:58 +0100 Subject: [PATCH 07/19] rtla/timerlat_top: Use BPF to collect samples Collect samples using BPF program instead of pulling them from tracefs. If the osnoise:timerlat_sample tracepoint is unavailable or the BPF program fails to load for whatever reason, rtla falls back to the old implementation. The collection of samples using the BPF program is fully self-contained and requires no activity of the userspace part of rtla during the measurement. Thus, rtla only pulls the summary from the BPF map and displays it every second, improving the performance. In --aa-only mode, the BPF program does not collect any data and only signalizes the end of tracing to userspace. An optimization that re-used the main trace instance for auto-analysis in aa-only mode was dropped, as rtla no longer turns tracing on in the main trace instance, making it useless for auto-analysis. Cc: John Kacur Cc: Luis Goncalves Cc: Gabriele Monaco Cc: Clark Williams Link: https://lore.kernel.org/20250218145859.27762-8-tglozar@redhat.com Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/timerlat_top.c | 255 ++++++++++++++++++++++---- 1 file changed, 215 insertions(+), 40 deletions(-) diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index a3d9e8f67a4f7..1b6455b9e0939 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -18,6 +18,7 @@ #include "timerlat.h" #include "timerlat_aa.h" #include "timerlat_u.h" +#include "timerlat_bpf.h" struct timerlat_top_cpu { unsigned long long irq_count; @@ -181,6 +182,76 @@ timerlat_top_handler(struct trace_seq *s, struct tep_record *record, return 0; } +/* + * timerlat_top_bpf_pull_data - copy data from BPF maps into userspace + */ +static int timerlat_top_bpf_pull_data(struct osnoise_tool *tool) +{ + struct timerlat_top_data *data = tool->data; + int i, err; + long long value_irq[data->nr_cpus], + value_thread[data->nr_cpus], + value_user[data->nr_cpus]; + + /* Pull summary */ + err = timerlat_bpf_get_summary_value(SUMMARY_CURRENT, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->cpu_data[i].cur_irq = value_irq[i]; + data->cpu_data[i].cur_thread = value_thread[i]; + data->cpu_data[i].cur_user = value_user[i]; + } + + err = timerlat_bpf_get_summary_value(SUMMARY_COUNT, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->cpu_data[i].irq_count = value_irq[i]; + data->cpu_data[i].thread_count = value_thread[i]; + data->cpu_data[i].user_count = value_user[i]; + } + + err = timerlat_bpf_get_summary_value(SUMMARY_MIN, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->cpu_data[i].min_irq = value_irq[i]; + data->cpu_data[i].min_thread = value_thread[i]; + data->cpu_data[i].min_user = value_user[i]; + } + + err = timerlat_bpf_get_summary_value(SUMMARY_MAX, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->cpu_data[i].max_irq = value_irq[i]; + data->cpu_data[i].max_thread = value_thread[i]; + data->cpu_data[i].max_user = value_user[i]; + } + + err = timerlat_bpf_get_summary_value(SUMMARY_SUM, + value_irq, value_thread, value_user, + data->nr_cpus); + if (err) + return err; + for (i = 0; i < data->nr_cpus; i++) { + data->cpu_data[i].sum_irq = value_irq[i]; + data->cpu_data[i].sum_thread = value_thread[i]; + data->cpu_data[i].sum_user = value_user[i]; + } + + return 0; +} + /* * timerlat_top_header - print the header of the tool output */ @@ -894,6 +965,111 @@ timerlat_top_set_signals(struct timerlat_params *params) } } +/* + * timerlat_top_main_loop - main loop to process events + */ +static int +timerlat_top_main_loop(struct osnoise_tool *top, + struct osnoise_tool *record, + struct timerlat_params *params, + struct timerlat_u_params *params_u) +{ + struct trace_instance *trace = &top->trace; + int retval; + + while (!stop_tracing) { + sleep(params->sleep_time); + + if (params->aa_only && !osnoise_trace_is_off(top, record)) + continue; + + retval = tracefs_iterate_raw_events(trace->tep, + trace->inst, + NULL, + 0, + collect_registered_events, + trace); + if (retval < 0) { + err_msg("Error iterating on events\n"); + return retval; + } + + if (!params->quiet) + timerlat_print_stats(params, top); + + if (osnoise_trace_is_off(top, record)) + break; + + /* is there still any user-threads ? */ + if (params->user_workload) { + if (params_u->stopped_running) { + debug_msg("timerlat user space threads stopped!\n"); + break; + } + } + } + + return 0; +} + +/* + * timerlat_top_bpf_main_loop - main loop to process events (BPF variant) + */ +static int +timerlat_top_bpf_main_loop(struct osnoise_tool *top, + struct osnoise_tool *record, + struct timerlat_params *params, + struct timerlat_u_params *params_u) +{ + int retval, wait_retval; + + if (params->aa_only) { + /* Auto-analysis only, just wait for stop tracing */ + timerlat_bpf_wait(-1); + return 0; + } + + if (params->quiet) { + /* Quiet mode: wait for stop and then, print results */ + timerlat_bpf_wait(-1); + + retval = timerlat_top_bpf_pull_data(top); + if (retval) { + err_msg("Error pulling BPF data\n"); + return retval; + } + + return 0; + } + + /* Pull and display data in a loop */ + while (!stop_tracing) { + wait_retval = timerlat_bpf_wait(params->sleep_time); + + retval = timerlat_top_bpf_pull_data(top); + if (retval) { + err_msg("Error pulling BPF data\n"); + return retval; + } + + timerlat_print_stats(params, top); + + if (wait_retval == 1) + /* Stopping requested by tracer */ + break; + + /* is there still any user-threads ? */ + if (params->user_workload) { + if (params_u->stopped_running) { + debug_msg("timerlat user space threads stopped!\n"); + break; + } + } + } + + return 0; +} + int timerlat_top_main(int argc, char *argv[]) { struct timerlat_params *params; @@ -908,6 +1084,7 @@ int timerlat_top_main(int argc, char *argv[]) char *max_lat; int retval; int nr_cpus, i; + bool no_bpf = false; params = timerlat_top_parse_args(argc, argv); if (!params) @@ -933,6 +1110,23 @@ int timerlat_top_main(int argc, char *argv[]) */ top_inst = trace; + if (getenv("RTLA_NO_BPF") && strncmp(getenv("RTLA_NO_BPF"), "1", 2) == 0) { + debug_msg("RTLA_NO_BPF set, disabling BPF\n"); + no_bpf = true; + } + + if (!no_bpf && !tep_find_event_by_name(trace->tep, "osnoise", "timerlat_sample")) { + debug_msg("osnoise:timerlat_sample missing, disabling BPF\n"); + no_bpf = true; + } + + if (!no_bpf) { + retval = timerlat_bpf_init(params); + if (retval) { + debug_msg("Could not enable BPF\n"); + no_bpf = true; + } + } retval = enable_timerlat(trace); if (retval) { @@ -1007,15 +1201,9 @@ int timerlat_top_main(int argc, char *argv[]) } if (!params->no_aa) { - if (params->aa_only) { - /* as top is not used for display, use it for aa */ - aa = top; - } else { - /* otherwise, a new instance is needed */ - aa = osnoise_init_tool("timerlat_aa"); - if (!aa) - goto out_top; - } + aa = osnoise_init_tool("timerlat_aa"); + if (!aa) + goto out_top; retval = timerlat_aa_init(aa, params->dump_tasks); if (retval) { @@ -1066,44 +1254,31 @@ int timerlat_top_main(int argc, char *argv[]) */ if (params->trace_output) trace_instance_start(&record->trace); - if (!params->no_aa && aa != top) + if (!params->no_aa) trace_instance_start(&aa->trace); - trace_instance_start(trace); + if (no_bpf) { + trace_instance_start(trace); + } else { + retval = timerlat_bpf_attach(); + if (retval) { + err_msg("Error attaching BPF program\n"); + goto out_top; + } + } top->start_time = time(NULL); timerlat_top_set_signals(params); - while (!stop_tracing) { - sleep(params->sleep_time); + if (no_bpf) + retval = timerlat_top_main_loop(top, record, params, ¶ms_u); + else + retval = timerlat_top_bpf_main_loop(top, record, params, ¶ms_u); - if (params->aa_only && !osnoise_trace_is_off(top, record)) - continue; + if (retval) + goto out_top; - retval = tracefs_iterate_raw_events(trace->tep, - trace->inst, - NULL, - 0, - collect_registered_events, - trace); - if (retval < 0) { - err_msg("Error iterating on events\n"); - goto out_top; - } - - if (!params->quiet) - timerlat_print_stats(params, top); - - if (osnoise_trace_is_off(top, record)) - break; - - /* is there still any user-threads ? */ - if (params->user_workload) { - if (params_u.stopped_running) { - debug_msg("timerlat user space threads stopped!\n"); - break; - } - } - } + if (!no_bpf) + timerlat_bpf_detach(); if (params->user_workload && !params_u.stopped_running) { params_u.should_run = 0; From 005682b403c5ecf7e1eb965e8dc9d6058061ee30 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Tue, 18 Feb 2025 15:58:59 +0100 Subject: [PATCH 08/19] rtla/timerlat: Test BPF mode Using the RTLA_NO_BPF environmental variable, execute rtla-timerlat tests both with and without BPF support to cover both paths. If rtla is built without BPF or the osnoise:timerlat_sample trace event is not available, test only the non-BPF path. Cc: John Kacur Cc: Luis Goncalves Cc: Gabriele Monaco Cc: Clark Williams Link: https://lore.kernel.org/20250218145859.27762-9-tglozar@redhat.com Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/tests/timerlat.t | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/tracing/rtla/tests/timerlat.t b/tools/tracing/rtla/tests/timerlat.t index e86f40e5749e5..e939ff71d6be3 100644 --- a/tools/tracing/rtla/tests/timerlat.t +++ b/tools/tracing/rtla/tests/timerlat.t @@ -4,7 +4,20 @@ source tests/engine.sh test_begin set_timeout 2m +timerlat_sample_event='/sys/kernel/tracing/events/osnoise/timerlat_sample' +if ldd $RTLA | grep libbpf >/dev/null && [ -d "$timerlat_sample_event" ] +then + # rtla build with BPF and system supports BPF mode + no_bpf_options='0 1' +else + no_bpf_options='1' +fi + +# Do every test with and without BPF +for option in $no_bpf_options +do +export RTLA_NO_BPF=$option check "verify help page" \ "timerlat --help" check "verify -s/--stack" \ @@ -23,5 +36,6 @@ check "verify -c/--cpus" \ "timerlat hist -c 0 -d 30s" check "hist test in nanoseconds" \ "timerlat hist -i 2 -c 0 -n -d 30s" +done test_end From e82c78afa3d48f6512570e7d39258cd68e7bae0a Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Tue, 4 Mar 2025 15:22:28 +0100 Subject: [PATCH 09/19] tools/rv: Keep user LDFLAGS in build rv, unlike rtla and perf, drops LDFLAGS supplied by the user and honors only EXTRA_LDFLAGS. This is inconsistent with both perf and rtla and can lead to all kinds of unexpected behavior. For example, on Fedora and RHEL, it causes rv to be build without PIE, unlike the aforementioned perf and rtla: $ file /usr/bin/{rv,rtla,perf} /usr/bin/rv: ELF 64-bit LSB executable, ... /usr/bin/rtla: ELF 64-bit LSB pie executable, ... /usr/bin/perf: ELF 64-bit LSB pie executable, ... Keep both LDFLAGS and EXTRA_LDFLAGS for the build. Cc: John Kacur Cc: Luis Goncalves Cc: Gabriele Monaco Link: https://lore.kernel.org/20250304142228.767658-1-tglozar@redhat.com Fixes: 012e4e77df73 ("tools/verification: Use tools/build makefiles on rv") Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/verification/rv/Makefile.rv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/verification/rv/Makefile.rv b/tools/verification/rv/Makefile.rv index 161baa29eb86c..2497fb96c83d2 100644 --- a/tools/verification/rv/Makefile.rv +++ b/tools/verification/rv/Makefile.rv @@ -27,7 +27,7 @@ endif INCLUDE := -Iinclude/ CFLAGS := -g -DVERSION=\"$(VERSION)\" $(FOPTS) $(WOPTS) $(EXTRA_CFLAGS) $(INCLUDE) -LDFLAGS := -ggdb $(EXTRA_LDFLAGS) +LDFLAGS := -ggdb $(LDFLAGS) $(EXTRA_LDFLAGS) INSTALL := install MKDIR := mkdir From dc4d4e7c72d17e6f6bdd9ce0d7a72a085828987f Mon Sep 17 00:00:00 2001 From: Costa Shulyupin Date: Wed, 19 Feb 2025 13:51:10 +0200 Subject: [PATCH 10/19] rtla: Refactor save_trace_to_file The functions osnoise_hist_main(), osnoise_top_main(), timerlat_hist_main(), and timerlat_top_main() are lengthy and contain duplicated code. Refactor by consolidating the duplicate lines into the save_trace_to_file() function. Cc: Daniel Bristot de Oliveira Cc: John Kacur Cc: "Luis Claudio R. Goncalves" Cc: Eder Zulian Cc: Dan Carpenter Cc: Gabriele Monaco Link: https://lore.kernel.org/20250219115138.406075-1-costa.shul@redhat.com Signed-off-by: Costa Shulyupin Reviewed-by: Tomas Glozar Tested-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/osnoise_hist.c | 5 +---- tools/tracing/rtla/src/osnoise_top.c | 5 +---- tools/tracing/rtla/src/timerlat_hist.c | 5 +---- tools/tracing/rtla/src/timerlat_top.c | 5 +---- tools/tracing/rtla/src/trace.c | 4 ++++ 5 files changed, 8 insertions(+), 16 deletions(-) diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index b4930b835b0a1..7c6ef67ef3e6c 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c @@ -983,10 +983,7 @@ int osnoise_hist_main(int argc, char *argv[]) if (osnoise_trace_is_off(tool, record)) { printf("rtla osnoise hit stop tracing\n"); - if (params->trace_output) { - printf(" Saving trace to %s\n", params->trace_output); - save_trace_to_file(record->trace.inst, params->trace_output); - } + save_trace_to_file(record->trace.inst, params->trace_output); } out_hist: diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index 4772677ac762c..0eeefbbbf3173 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c @@ -813,10 +813,7 @@ int osnoise_top_main(int argc, char **argv) if (osnoise_trace_is_off(tool, record)) { printf("osnoise hit stop tracing\n"); - if (params->trace_output) { - printf(" Saving trace to %s\n", params->trace_output); - save_trace_to_file(record->trace.inst, params->trace_output); - } + save_trace_to_file(record->trace.inst, params->trace_output); } out_top: diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index 202d99a598ba8..93d0c9e450204 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -1473,10 +1473,7 @@ int timerlat_hist_main(int argc, char *argv[]) if (!params->no_aa) timerlat_auto_analysis(params->stop_us, params->stop_total_us); - if (params->trace_output) { - printf(" Saving trace to %s\n", params->trace_output); - save_trace_to_file(record->trace.inst, params->trace_output); - } + save_trace_to_file(record->trace.inst, params->trace_output); } out_hist: diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 1b6455b9e0939..3894ac37d81ca 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -1295,10 +1295,7 @@ int timerlat_top_main(int argc, char *argv[]) if (!params->no_aa) timerlat_auto_analysis(params->stop_us, params->stop_total_us); - if (params->trace_output) { - printf(" Saving trace to %s\n", params->trace_output); - save_trace_to_file(record->trace.inst, params->trace_output); - } + save_trace_to_file(record->trace.inst, params->trace_output); } else if (params->aa_only) { /* * If the trace did not stop with --aa-only, at least print the diff --git a/tools/tracing/rtla/src/trace.c b/tools/tracing/rtla/src/trace.c index 728f5029d5335..74ed2f6208baa 100644 --- a/tools/tracing/rtla/src/trace.c +++ b/tools/tracing/rtla/src/trace.c @@ -75,12 +75,16 @@ int save_trace_to_file(struct tracefs_instance *inst, const char *filename) int out_fd, in_fd; int retval = -1; + if (!filename) + return 0; + in_fd = tracefs_instance_file_open(inst, file, O_RDONLY); if (in_fd < 0) { err_msg("Failed to open trace file\n"); return -1; } + printf(" Saving trace to %s\n", filename); out_fd = creat(filename, mode); if (out_fd < 0) { err_msg("Failed to create output file %s\n", filename); From 814d051ebed40b27285ab3c5e2454bd01a0f9631 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Wed, 26 Mar 2025 01:40:18 +0100 Subject: [PATCH 11/19] tools/build: Use SYSTEM_BPFTOOL for system bpftool The feature test for system bpftool uses BPFTOOL as the variable to set its path, defaulting to just "bpftool" if not set by the user. This conflicts with selftests and a few other utilities, which expect BPFTOOL to be set to the in-tree bpftool path by default. For example, bpftool selftests fail to build: $ make -C tools/testing/selftests/bpf/ make: Entering directory '/home/tglozar/dev/linux/tools/testing/selftests/bpf' make: *** No rule to make target 'bpftool', needed by '/home/tglozar/dev/linux/tools/testing/selftests/bpf/tools/include/vmlinux.h'. Stop. make: Leaving directory '/home/tglozar/dev/linux/tools/testing/selftests/bpf' Fix the problem by renaming the variable used for system bpftool from BPFTOOL to SYSTEM_BPFTOOL, so that the new usage does not conflict with the existing one of BPFTOOL. Cc: John Kacur Cc: Luis Goncalves Link: https://lore.kernel.org/20250326004018.248357-1-tglozar@redhat.com Fixes: 8a635c3856dd ("tools/build: Add bpftool-skeletons feature test") Closes: https://lore.kernel.org/linux-kernel/5df6968a-2e5f-468e-b457-fc201535dd4c@linux.ibm.com/ Reported-by: Venkat Rao Bagalkote Tested-by: Venkat Rao Bagalkote Suggested-by: Quentin Monnet Acked-by: Quentin Monnet Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/build/feature/Makefile | 2 +- tools/scripts/Makefile.include | 2 +- tools/tracing/rtla/Makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 4f9c1d950f5db..b8b5fb183dd40 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -419,7 +419,7 @@ $(OUTPUT)test-libpfm4.bin: $(BUILD) -lpfm $(OUTPUT)test-bpftool-skeletons.bin: - $(BPFTOOL) version | grep '^features:.*skeletons' \ + $(SYSTEM_BPFTOOL) version | grep '^features:.*skeletons' \ > $(@:.bin=.make.output) 2>&1 ############################### diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include index 71bbe52721b35..34a33c1ad96cf 100644 --- a/tools/scripts/Makefile.include +++ b/tools/scripts/Makefile.include @@ -92,7 +92,7 @@ LLVM_OBJCOPY ?= llvm-objcopy LLVM_STRIP ?= llvm-strip # Some tools require bpftool -BPFTOOL ?= bpftool +SYSTEM_BPFTOOL ?= bpftool ifeq ($(CC_NO_CLANG), 1) EXTRA_WARNINGS += -Wstrict-aliasing=3 diff --git a/tools/tracing/rtla/Makefile b/tools/tracing/rtla/Makefile index 557af322be610..3dc050317b9d4 100644 --- a/tools/tracing/rtla/Makefile +++ b/tools/tracing/rtla/Makefile @@ -78,7 +78,7 @@ src/timerlat.bpf.o: src/timerlat.bpf.c $(QUIET_CLANG)$(CLANG) -g -O2 -target bpf -c $(filter %.c,$^) -o $@ src/timerlat.skel.h: src/timerlat.bpf.o - $(QUIET_GENSKEL)$(BPFTOOL) gen skeleton $< > $@ + $(QUIET_GENSKEL)$(SYSTEM_BPFTOOL) gen skeleton $< > $@ else src/timerlat.skel.h: $(Q)echo '/* BPF skeleton is disabled */' > src/timerlat.skel.h From c57c58a62e564c615520839742b28d315427a280 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Thu, 13 Mar 2025 15:10:34 +0100 Subject: [PATCH 12/19] rtla: Fix segfault in save_trace_to_file call Running rtla with exit on threshold, but without saving trace leads to a segmenetation fault: $ rtla timerlat hist -T 10 ... Max timerlat IRQ latency from idle: 4.29 us in cpu 0 Segmentation fault This is caused by null pointer deference in the call of save_trace_to_file, which attempts to dereference an uninitialized osnoise_tool variable: save_trace_to_file(record->trace.inst, params->trace_output); ^ this is uninitialized if params->trace_output is not set Fix this by not attempting to dereference "record" if it is NULL and passing NULL instead. As a safety measure, the first field is also checked for NULL inside save_trace_to_file. Cc: John Kacur Cc: Luis Goncalves Cc: Costa Shulyupin Link: https://lore.kernel.org/20250313141034.299117-1-tglozar@redhat.com Fixes: dc4d4e7c72d1 ("rtla: Refactor save_trace_to_file") Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/osnoise_hist.c | 3 ++- tools/tracing/rtla/src/osnoise_top.c | 3 ++- tools/tracing/rtla/src/timerlat_hist.c | 3 ++- tools/tracing/rtla/src/timerlat_top.c | 3 ++- tools/tracing/rtla/src/trace.c | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index 7c6ef67ef3e6c..f4c9051c33c4d 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c @@ -983,7 +983,8 @@ int osnoise_hist_main(int argc, char *argv[]) if (osnoise_trace_is_off(tool, record)) { printf("rtla osnoise hit stop tracing\n"); - save_trace_to_file(record->trace.inst, params->trace_output); + save_trace_to_file(record ? record->trace.inst : NULL, + params->trace_output); } out_hist: diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index 0eeefbbbf3173..dacec2f990177 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c @@ -813,7 +813,8 @@ int osnoise_top_main(int argc, char **argv) if (osnoise_trace_is_off(tool, record)) { printf("osnoise hit stop tracing\n"); - save_trace_to_file(record->trace.inst, params->trace_output); + save_trace_to_file(record ? record->trace.inst : NULL, + params->trace_output); } out_top: diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index 93d0c9e450204..822c068b47767 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -1473,7 +1473,8 @@ int timerlat_hist_main(int argc, char *argv[]) if (!params->no_aa) timerlat_auto_analysis(params->stop_us, params->stop_total_us); - save_trace_to_file(record->trace.inst, params->trace_output); + save_trace_to_file(record ? record->trace.inst : NULL, + params->trace_output); } out_hist: diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 3894ac37d81ca..c3196a0bb5851 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -1295,7 +1295,8 @@ int timerlat_top_main(int argc, char *argv[]) if (!params->no_aa) timerlat_auto_analysis(params->stop_us, params->stop_total_us); - save_trace_to_file(record->trace.inst, params->trace_output); + save_trace_to_file(record ? record->trace.inst : NULL, + params->trace_output); } else if (params->aa_only) { /* * If the trace did not stop with --aa-only, at least print the diff --git a/tools/tracing/rtla/src/trace.c b/tools/tracing/rtla/src/trace.c index 74ed2f6208baa..69cbc48d53d3a 100644 --- a/tools/tracing/rtla/src/trace.c +++ b/tools/tracing/rtla/src/trace.c @@ -75,7 +75,7 @@ int save_trace_to_file(struct tracefs_instance *inst, const char *filename) int out_fd, in_fd; int retval = -1; - if (!filename) + if (!inst || !filename) return 0; in_fd = tracefs_instance_file_open(inst, file, O_RDONLY); From 025b217990ea5cdc24a33ad60c889c696cbf9a32 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Thu, 20 Mar 2025 10:24:55 +0100 Subject: [PATCH 13/19] rtla/osnoise: Unify params struct Instead of having separate structs osnoise_top_params and osnoise_hist_params, use one struct osnoise_params for both. This allows code using the structs to be shared between osnoise-top and osnoise-hist. Cc: Luis Goncalves Link: https://lore.kernel.org/20250320092500.101385-2-tglozar@redhat.com Signed-off-by: Tomas Glozar Reviewed-by: John Kacur Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/osnoise.c | 1 - tools/tracing/rtla/src/osnoise.h | 47 +++++++++++++++++++++++ tools/tracing/rtla/src/osnoise_hist.c | 52 ++++++-------------------- tools/tracing/rtla/src/osnoise_top.c | 54 +++++---------------------- tools/tracing/rtla/src/timerlat.h | 1 - 5 files changed, 68 insertions(+), 87 deletions(-) diff --git a/tools/tracing/rtla/src/osnoise.c b/tools/tracing/rtla/src/osnoise.c index 85f398b89597a..93d485c0e9499 100644 --- a/tools/tracing/rtla/src/osnoise.c +++ b/tools/tracing/rtla/src/osnoise.c @@ -14,7 +14,6 @@ #include #include "osnoise.h" -#include "utils.h" /* * osnoise_get_cpus - return the original "osnoise/cpus" content diff --git a/tools/tracing/rtla/src/osnoise.h b/tools/tracing/rtla/src/osnoise.h index 056c8b113dee6..f78ffbdc8c8d8 100644 --- a/tools/tracing/rtla/src/osnoise.h +++ b/tools/tracing/rtla/src/osnoise.h @@ -1,8 +1,55 @@ // SPDX-License-Identifier: GPL-2.0 #pragma once +#include "utils.h" #include "trace.h" +enum osnoise_mode { + MODE_OSNOISE = 0, + MODE_HWNOISE +}; + +struct osnoise_params { + /* Common params */ + char *cpus; + cpu_set_t monitored_cpus; + char *trace_output; + char *cgroup_name; + unsigned long long runtime; + unsigned long long period; + long long threshold; + long long stop_us; + long long stop_total_us; + int sleep_time; + int duration; + int set_sched; + int cgroup; + int hk_cpus; + cpu_set_t hk_cpu_set; + struct sched_attr sched_param; + struct trace_events *events; + int warmup; + int buffer_size; + union { + struct { + /* top only */ + int quiet; + int pretty_output; + enum osnoise_mode mode; + }; + struct { + /* hist only */ + int output_divisor; + char no_header; + char no_summary; + char no_index; + char with_zeros; + int bucket_size; + int entries; + }; + }; +}; + /* * osnoise_context - read, store, write, restore osnoise configs. */ diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index f4c9051c33c4d..4721f15f77cd0 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c @@ -14,38 +14,8 @@ #include #include -#include "utils.h" #include "osnoise.h" -struct osnoise_hist_params { - char *cpus; - cpu_set_t monitored_cpus; - char *trace_output; - char *cgroup_name; - unsigned long long runtime; - unsigned long long period; - long long threshold; - long long stop_us; - long long stop_total_us; - int sleep_time; - int duration; - int set_sched; - int output_divisor; - int cgroup; - int hk_cpus; - cpu_set_t hk_cpu_set; - struct sched_attr sched_param; - struct trace_events *events; - char no_header; - char no_summary; - char no_index; - char with_zeros; - int bucket_size; - int entries; - int warmup; - int buffer_size; -}; - struct osnoise_hist_cpu { int *samples; int count; @@ -126,7 +96,7 @@ static struct osnoise_hist_data static void osnoise_hist_update_multiple(struct osnoise_tool *tool, int cpu, unsigned long long duration, int count) { - struct osnoise_hist_params *params = tool->params; + struct osnoise_params *params = tool->params; struct osnoise_hist_data *data = tool->data; unsigned long long total_duration; int entries = data->entries; @@ -168,7 +138,7 @@ static void osnoise_destroy_trace_hist(struct osnoise_tool *tool) */ static int osnoise_init_trace_hist(struct osnoise_tool *tool) { - struct osnoise_hist_params *params = tool->params; + struct osnoise_params *params = tool->params; struct osnoise_hist_data *data = tool->data; int bucket_size; char buff[128]; @@ -253,7 +223,7 @@ static void osnoise_read_trace_hist(struct osnoise_tool *tool) */ static void osnoise_hist_header(struct osnoise_tool *tool) { - struct osnoise_hist_params *params = tool->params; + struct osnoise_params *params = tool->params; struct osnoise_hist_data *data = tool->data; struct trace_seq *s = tool->trace.seq; char duration[26]; @@ -292,7 +262,7 @@ static void osnoise_hist_header(struct osnoise_tool *tool) * osnoise_print_summary - print the summary of the hist data to the output */ static void -osnoise_print_summary(struct osnoise_hist_params *params, +osnoise_print_summary(struct osnoise_params *params, struct trace_instance *trace, struct osnoise_hist_data *data) { @@ -370,7 +340,7 @@ osnoise_print_summary(struct osnoise_hist_params *params, * osnoise_print_stats - print data for all CPUs */ static void -osnoise_print_stats(struct osnoise_hist_params *params, struct osnoise_tool *tool) +osnoise_print_stats(struct osnoise_params *params, struct osnoise_tool *tool) { struct osnoise_hist_data *data = tool->data; struct trace_instance *trace = &tool->trace; @@ -508,10 +478,10 @@ static void osnoise_hist_usage(char *usage) /* * osnoise_hist_parse_args - allocs, parse and fill the cmd line parameters */ -static struct osnoise_hist_params +static struct osnoise_params *osnoise_hist_parse_args(int argc, char *argv[]) { - struct osnoise_hist_params *params; + struct osnoise_params *params; struct trace_events *tevent; int retval; int c; @@ -731,7 +701,7 @@ static struct osnoise_hist_params * osnoise_hist_apply_config - apply the hist configs to the initialized tool */ static int -osnoise_hist_apply_config(struct osnoise_tool *tool, struct osnoise_hist_params *params) +osnoise_hist_apply_config(struct osnoise_tool *tool, struct osnoise_params *params) { int retval; @@ -808,7 +778,7 @@ osnoise_hist_apply_config(struct osnoise_tool *tool, struct osnoise_hist_params * osnoise_init_hist - initialize a osnoise hist tool with parameters */ static struct osnoise_tool -*osnoise_init_hist(struct osnoise_hist_params *params) +*osnoise_init_hist(struct osnoise_params *params) { struct osnoise_tool *tool; int nr_cpus; @@ -842,7 +812,7 @@ static void stop_hist(int sig) * osnoise_hist_set_signals - handles the signal to stop the tool */ static void -osnoise_hist_set_signals(struct osnoise_hist_params *params) +osnoise_hist_set_signals(struct osnoise_params *params) { signal(SIGINT, stop_hist); if (params->duration) { @@ -853,7 +823,7 @@ osnoise_hist_set_signals(struct osnoise_hist_params *params) int osnoise_hist_main(int argc, char *argv[]) { - struct osnoise_hist_params *params; + struct osnoise_params *params; struct osnoise_tool *record = NULL; struct osnoise_tool *tool = NULL; struct trace_instance *trace; diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index dacec2f990177..7f393019bbf59 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c @@ -14,40 +14,6 @@ #include #include "osnoise.h" -#include "utils.h" - -enum osnoise_mode { - MODE_OSNOISE = 0, - MODE_HWNOISE -}; - -/* - * osnoise top parameters - */ -struct osnoise_top_params { - char *cpus; - cpu_set_t monitored_cpus; - char *trace_output; - char *cgroup_name; - unsigned long long runtime; - unsigned long long period; - long long threshold; - long long stop_us; - long long stop_total_us; - int sleep_time; - int duration; - int quiet; - int set_sched; - int cgroup; - int hk_cpus; - int warmup; - int buffer_size; - int pretty_output; - cpu_set_t hk_cpu_set; - struct sched_attr sched_param; - struct trace_events *events; - enum osnoise_mode mode; -}; struct osnoise_top_cpu { unsigned long long sum_runtime; @@ -158,7 +124,7 @@ osnoise_top_handler(struct trace_seq *s, struct tep_record *record, */ static void osnoise_top_header(struct osnoise_tool *top) { - struct osnoise_top_params *params = top->params; + struct osnoise_params *params = top->params; struct trace_seq *s = top->trace.seq; char duration[26]; @@ -218,7 +184,7 @@ static void clear_terminal(struct trace_seq *seq) */ static void osnoise_top_print(struct osnoise_tool *tool, int cpu) { - struct osnoise_top_params *params = tool->params; + struct osnoise_params *params = tool->params; struct trace_seq *s = tool->trace.seq; struct osnoise_top_cpu *cpu_data; struct osnoise_top_data *data; @@ -258,7 +224,7 @@ static void osnoise_top_print(struct osnoise_tool *tool, int cpu) * osnoise_print_stats - print data for all cpus */ static void -osnoise_print_stats(struct osnoise_top_params *params, struct osnoise_tool *top) +osnoise_print_stats(struct osnoise_params *params, struct osnoise_tool *top) { struct trace_instance *trace = &top->trace; static int nr_cpus = -1; @@ -286,7 +252,7 @@ osnoise_print_stats(struct osnoise_top_params *params, struct osnoise_tool *top) /* * osnoise_top_usage - prints osnoise top usage message */ -static void osnoise_top_usage(struct osnoise_top_params *params, char *usage) +static void osnoise_top_usage(struct osnoise_params *params, char *usage) { int i; @@ -354,9 +320,9 @@ static void osnoise_top_usage(struct osnoise_top_params *params, char *usage) /* * osnoise_top_parse_args - allocs, parse and fill the cmd line parameters */ -struct osnoise_top_params *osnoise_top_parse_args(int argc, char **argv) +struct osnoise_params *osnoise_top_parse_args(int argc, char **argv) { - struct osnoise_top_params *params; + struct osnoise_params *params; struct trace_events *tevent; int retval; int c; @@ -553,7 +519,7 @@ struct osnoise_top_params *osnoise_top_parse_args(int argc, char **argv) * osnoise_top_apply_config - apply the top configs to the initialized tool */ static int -osnoise_top_apply_config(struct osnoise_tool *tool, struct osnoise_top_params *params) +osnoise_top_apply_config(struct osnoise_tool *tool, struct osnoise_params *params) { int retval; @@ -640,7 +606,7 @@ osnoise_top_apply_config(struct osnoise_tool *tool, struct osnoise_top_params *p /* * osnoise_init_top - initialize a osnoise top tool with parameters */ -struct osnoise_tool *osnoise_init_top(struct osnoise_top_params *params) +struct osnoise_tool *osnoise_init_top(struct osnoise_params *params) { struct osnoise_tool *tool; int nr_cpus; @@ -674,7 +640,7 @@ static void stop_top(int sig) /* * osnoise_top_set_signals - handles the signal to stop the tool */ -static void osnoise_top_set_signals(struct osnoise_top_params *params) +static void osnoise_top_set_signals(struct osnoise_params *params) { signal(SIGINT, stop_top); if (params->duration) { @@ -685,7 +651,7 @@ static void osnoise_top_set_signals(struct osnoise_top_params *params) int osnoise_top_main(int argc, char **argv) { - struct osnoise_top_params *params; + struct osnoise_params *params; struct osnoise_tool *record = NULL; struct osnoise_tool *tool = NULL; struct trace_instance *trace; diff --git a/tools/tracing/rtla/src/timerlat.h b/tools/tracing/rtla/src/timerlat.h index e452d385cb0f4..cadc613dc82ef 100644 --- a/tools/tracing/rtla/src/timerlat.h +++ b/tools/tracing/rtla/src/timerlat.h @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -#include "utils.h" #include "osnoise.h" struct timerlat_params { From 20d6b07581ce37338cc709b4381e8f314309b187 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Thu, 20 Mar 2025 10:24:56 +0100 Subject: [PATCH 14/19] rtla: Unify apply_config between top and hist The functions osnoise_top_apply_config and osnoise_hist_apply_config, as well as timerlat_top_apply_config and timerlat_hist_apply_config, are mostly the same. Move common part from them into separate functions osnoise_apply_config and timerlat_apply_config. For rtla-timerlat, also unify params->user_hist and params->user_top into one field called params->user_data, and move several fields used only by timerlat-top into the top-only section of struct timerlat_params. Cc: Luis Goncalves Link: https://lore.kernel.org/20250320092500.101385-3-tglozar@redhat.com Signed-off-by: Tomas Glozar Reviewed-by: John Kacur Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/osnoise.c | 79 ++++++++++++++++ tools/tracing/rtla/src/osnoise.h | 1 + tools/tracing/rtla/src/osnoise_hist.c | 66 +------------- tools/tracing/rtla/src/osnoise_top.c | 66 +------------- tools/tracing/rtla/src/timerlat.c | 109 ++++++++++++++++++++++ tools/tracing/rtla/src/timerlat.h | 11 +-- tools/tracing/rtla/src/timerlat_hist.c | 119 ++++--------------------- tools/tracing/rtla/src/timerlat_top.c | 110 +++-------------------- 8 files changed, 227 insertions(+), 334 deletions(-) diff --git a/tools/tracing/rtla/src/osnoise.c b/tools/tracing/rtla/src/osnoise.c index 93d485c0e9499..1735a36466c4b 100644 --- a/tools/tracing/rtla/src/osnoise.c +++ b/tools/tracing/rtla/src/osnoise.c @@ -3,6 +3,7 @@ * Copyright (C) 2021 Red Hat Inc, Daniel Bristot de Oliveira */ +#define _GNU_SOURCE #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include "osnoise.h" @@ -1114,6 +1116,83 @@ osnoise_report_missed_events(struct osnoise_tool *tool) } } +/* + * osnoise_apply_config - apply common configs to the initialized tool + */ +int +osnoise_apply_config(struct osnoise_tool *tool, struct osnoise_params *params) +{ + int retval; + + if (!params->sleep_time) + params->sleep_time = 1; + + if (params->cpus) { + retval = osnoise_set_cpus(tool->context, params->cpus); + if (retval) { + err_msg("Failed to apply CPUs config\n"); + goto out_err; + } + } + + if (params->runtime || params->period) { + retval = osnoise_set_runtime_period(tool->context, + params->runtime, + params->period); + if (retval) { + err_msg("Failed to set runtime and/or period\n"); + goto out_err; + } + } + + if (params->stop_us) { + retval = osnoise_set_stop_us(tool->context, params->stop_us); + if (retval) { + err_msg("Failed to set stop us\n"); + goto out_err; + } + } + + if (params->stop_total_us) { + retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us); + if (retval) { + err_msg("Failed to set stop total us\n"); + goto out_err; + } + } + + if (params->threshold) { + retval = osnoise_set_tracing_thresh(tool->context, params->threshold); + if (retval) { + err_msg("Failed to set tracing_thresh\n"); + goto out_err; + } + } + + if (params->hk_cpus) { + retval = sched_setaffinity(getpid(), sizeof(params->hk_cpu_set), + ¶ms->hk_cpu_set); + if (retval == -1) { + err_msg("Failed to set rtla to the house keeping CPUs\n"); + goto out_err; + } + } else if (params->cpus) { + /* + * Even if the user do not set a house-keeping CPU, try to + * move rtla to a CPU set different to the one where the user + * set the workload to run. + * + * No need to check results as this is an automatic attempt. + */ + auto_house_keeping(¶ms->monitored_cpus); + } + + return 0; + +out_err: + return -1; +} + static void osnoise_usage(int err) { int i; diff --git a/tools/tracing/rtla/src/osnoise.h b/tools/tracing/rtla/src/osnoise.h index f78ffbdc8c8d8..ac1c999107440 100644 --- a/tools/tracing/rtla/src/osnoise.h +++ b/tools/tracing/rtla/src/osnoise.h @@ -155,6 +155,7 @@ struct osnoise_tool *osnoise_init_tool(char *tool_name); struct osnoise_tool *osnoise_init_trace_tool(char *tracer); void osnoise_report_missed_events(struct osnoise_tool *tool); bool osnoise_trace_is_off(struct osnoise_tool *tool, struct osnoise_tool *record); +int osnoise_apply_config(struct osnoise_tool *tool, struct osnoise_params *params); int osnoise_hist_main(int argc, char *argv[]); int osnoise_top_main(int argc, char **argv); diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index 4721f15f77cd0..d9d15c8f27c71 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c @@ -12,7 +12,6 @@ #include #include #include -#include #include "osnoise.h" @@ -705,68 +704,9 @@ osnoise_hist_apply_config(struct osnoise_tool *tool, struct osnoise_params *para { int retval; - if (!params->sleep_time) - params->sleep_time = 1; - - if (params->cpus) { - retval = osnoise_set_cpus(tool->context, params->cpus); - if (retval) { - err_msg("Failed to apply CPUs config\n"); - goto out_err; - } - } - - if (params->runtime || params->period) { - retval = osnoise_set_runtime_period(tool->context, - params->runtime, - params->period); - if (retval) { - err_msg("Failed to set runtime and/or period\n"); - goto out_err; - } - } - - if (params->stop_us) { - retval = osnoise_set_stop_us(tool->context, params->stop_us); - if (retval) { - err_msg("Failed to set stop us\n"); - goto out_err; - } - } - - if (params->stop_total_us) { - retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us); - if (retval) { - err_msg("Failed to set stop total us\n"); - goto out_err; - } - } - - if (params->threshold) { - retval = osnoise_set_tracing_thresh(tool->context, params->threshold); - if (retval) { - err_msg("Failed to set tracing_thresh\n"); - goto out_err; - } - } - - if (params->hk_cpus) { - retval = sched_setaffinity(getpid(), sizeof(params->hk_cpu_set), - ¶ms->hk_cpu_set); - if (retval == -1) { - err_msg("Failed to set rtla to the house keeping CPUs\n"); - goto out_err; - } - } else if (params->cpus) { - /* - * Even if the user do not set a house-keeping CPU, try to - * move rtla to a CPU set different to the one where the user - * set the workload to run. - * - * No need to check results as this is an automatic attempt. - */ - auto_house_keeping(¶ms->monitored_cpus); - } + retval = osnoise_apply_config(tool, params); + if (retval) + goto out_err; return 0; diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index 7f393019bbf59..3455ee73e2e69 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c @@ -11,7 +11,6 @@ #include #include #include -#include #include "osnoise.h" @@ -523,50 +522,9 @@ osnoise_top_apply_config(struct osnoise_tool *tool, struct osnoise_params *param { int retval; - if (!params->sleep_time) - params->sleep_time = 1; - - if (params->cpus) { - retval = osnoise_set_cpus(tool->context, params->cpus); - if (retval) { - err_msg("Failed to apply CPUs config\n"); - goto out_err; - } - } - - if (params->runtime || params->period) { - retval = osnoise_set_runtime_period(tool->context, - params->runtime, - params->period); - if (retval) { - err_msg("Failed to set runtime and/or period\n"); - goto out_err; - } - } - - if (params->stop_us) { - retval = osnoise_set_stop_us(tool->context, params->stop_us); - if (retval) { - err_msg("Failed to set stop us\n"); - goto out_err; - } - } - - if (params->stop_total_us) { - retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us); - if (retval) { - err_msg("Failed to set stop total us\n"); - goto out_err; - } - } - - if (params->threshold) { - retval = osnoise_set_tracing_thresh(tool->context, params->threshold); - if (retval) { - err_msg("Failed to set tracing_thresh\n"); - goto out_err; - } - } + retval = osnoise_apply_config(tool, params); + if (retval) + goto out_err; if (params->mode == MODE_HWNOISE) { retval = osnoise_set_irq_disable(tool->context, 1); @@ -576,24 +534,6 @@ osnoise_top_apply_config(struct osnoise_tool *tool, struct osnoise_params *param } } - if (params->hk_cpus) { - retval = sched_setaffinity(getpid(), sizeof(params->hk_cpu_set), - ¶ms->hk_cpu_set); - if (retval == -1) { - err_msg("Failed to set rtla to the house keeping CPUs\n"); - goto out_err; - } - } else if (params->cpus) { - /* - * Even if the user do not set a house-keeping CPU, try to - * move rtla to a CPU set different to the one where the user - * set the workload to run. - * - * No need to check results as this is an automatic attempt. - */ - auto_house_keeping(¶ms->monitored_cpus); - } - if (isatty(STDOUT_FILENO) && !params->quiet) params->pretty_output = 1; diff --git a/tools/tracing/rtla/src/timerlat.c b/tools/tracing/rtla/src/timerlat.c index 21cdcc5c4a296..448fb1f7d3a6a 100644 --- a/tools/tracing/rtla/src/timerlat.c +++ b/tools/tracing/rtla/src/timerlat.c @@ -2,6 +2,7 @@ /* * Copyright (C) 2021 Red Hat Inc, Daniel Bristot de Oliveira */ +#define _GNU_SOURCE #include #include #include @@ -11,9 +12,117 @@ #include #include #include +#include #include "timerlat.h" +/* + * timerlat_apply_config - apply common configs to the initialized tool + */ +int +timerlat_apply_config(struct osnoise_tool *tool, struct timerlat_params *params) +{ + int retval, i; + + if (!params->sleep_time) + params->sleep_time = 1; + + if (params->cpus) { + retval = osnoise_set_cpus(tool->context, params->cpus); + if (retval) { + err_msg("Failed to apply CPUs config\n"); + goto out_err; + } + } else { + for (i = 0; i < sysconf(_SC_NPROCESSORS_CONF); i++) + CPU_SET(i, ¶ms->monitored_cpus); + } + + if (params->stop_us) { + retval = osnoise_set_stop_us(tool->context, params->stop_us); + if (retval) { + err_msg("Failed to set stop us\n"); + goto out_err; + } + } + + if (params->stop_total_us) { + retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us); + if (retval) { + err_msg("Failed to set stop total us\n"); + goto out_err; + } + } + + + if (params->timerlat_period_us) { + retval = osnoise_set_timerlat_period_us(tool->context, params->timerlat_period_us); + if (retval) { + err_msg("Failed to set timerlat period\n"); + goto out_err; + } + } + + + if (params->print_stack) { + retval = osnoise_set_print_stack(tool->context, params->print_stack); + if (retval) { + err_msg("Failed to set print stack\n"); + goto out_err; + } + } + + if (params->hk_cpus) { + retval = sched_setaffinity(getpid(), sizeof(params->hk_cpu_set), + ¶ms->hk_cpu_set); + if (retval == -1) { + err_msg("Failed to set rtla to the house keeping CPUs\n"); + goto out_err; + } + } else if (params->cpus) { + /* + * Even if the user do not set a house-keeping CPU, try to + * move rtla to a CPU set different to the one where the user + * set the workload to run. + * + * No need to check results as this is an automatic attempt. + */ + auto_house_keeping(¶ms->monitored_cpus); + } + + /* + * If the user did not specify a type of thread, try user-threads first. + * Fall back to kernel threads otherwise. + */ + if (!params->kernel_workload && !params->user_data) { + retval = tracefs_file_exists(NULL, "osnoise/per_cpu/cpu0/timerlat_fd"); + if (retval) { + debug_msg("User-space interface detected, setting user-threads\n"); + params->user_workload = 1; + params->user_data = 1; + } else { + debug_msg("User-space interface not detected, setting kernel-threads\n"); + params->kernel_workload = 1; + } + } + + /* + * Set workload according to type of thread if the kernel supports it. + * On kernels without support, user threads will have already failed + * on missing timerlat_fd, and kernel threads do not need it. + */ + retval = osnoise_set_workload(tool->context, params->kernel_workload); + if (retval < -1) { + err_msg("Failed to set OSNOISE_WORKLOAD option\n"); + goto out_err; + } + + return 0; + +out_err: + return -1; +} + static void timerlat_usage(int err) { int i; diff --git a/tools/tracing/rtla/src/timerlat.h b/tools/tracing/rtla/src/timerlat.h index cadc613dc82ef..73045aef23fa0 100644 --- a/tools/tracing/rtla/src/timerlat.h +++ b/tools/tracing/rtla/src/timerlat.h @@ -15,17 +15,15 @@ struct timerlat_params { int sleep_time; int output_divisor; int duration; - int quiet; int set_sched; int dma_latency; int no_aa; - int aa_only; int dump_tasks; int cgroup; int hk_cpus; int user_workload; int kernel_workload; - int pretty_output; + int user_data; int warmup; int buffer_size; int deepest_idle_state; @@ -35,11 +33,12 @@ struct timerlat_params { union { struct { /* top only */ - int user_top; + int quiet; + int aa_only; + int pretty_output; }; struct { /* hist only */ - int user_hist; char no_irq; char no_thread; char no_header; @@ -52,6 +51,8 @@ struct timerlat_params { }; }; +int timerlat_apply_config(struct osnoise_tool *tool, struct timerlat_params *params); + int timerlat_hist_main(int argc, char *argv[]); int timerlat_top_main(int argc, char *argv[]); int timerlat_main(int argc, char *argv[]); diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index 822c068b47767..9d9efeedc4c2c 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -315,7 +315,7 @@ static void timerlat_hist_header(struct osnoise_tool *tool) if (!params->no_thread) trace_seq_printf(s, " Thr-%03d", cpu); - if (params->user_hist) + if (params->user_data) trace_seq_printf(s, " Usr-%03d", cpu); } trace_seq_printf(s, "\n"); @@ -371,7 +371,7 @@ timerlat_print_summary(struct timerlat_params *params, trace_seq_printf(trace->seq, "%9llu ", data->hist[cpu].thread_count); - if (params->user_hist) + if (params->user_data) trace_seq_printf(trace->seq, "%9llu ", data->hist[cpu].user_count); } @@ -399,7 +399,7 @@ timerlat_print_summary(struct timerlat_params *params, data->hist[cpu].min_thread, false); - if (params->user_hist) + if (params->user_data) format_summary_value(trace->seq, data->hist[cpu].user_count, data->hist[cpu].min_user, @@ -429,7 +429,7 @@ timerlat_print_summary(struct timerlat_params *params, data->hist[cpu].sum_thread, true); - if (params->user_hist) + if (params->user_data) format_summary_value(trace->seq, data->hist[cpu].user_count, data->hist[cpu].sum_user, @@ -459,7 +459,7 @@ timerlat_print_summary(struct timerlat_params *params, data->hist[cpu].max_thread, false); - if (params->user_hist) + if (params->user_data) format_summary_value(trace->seq, data->hist[cpu].user_count, data->hist[cpu].max_user, @@ -521,7 +521,7 @@ timerlat_print_stats_all(struct timerlat_params *params, if (!params->no_thread) trace_seq_printf(trace->seq, " Thr"); - if (params->user_hist) + if (params->user_data) trace_seq_printf(trace->seq, " Usr"); trace_seq_printf(trace->seq, "\n"); @@ -537,7 +537,7 @@ timerlat_print_stats_all(struct timerlat_params *params, trace_seq_printf(trace->seq, "%9llu ", sum.thread_count); - if (params->user_hist) + if (params->user_data) trace_seq_printf(trace->seq, "%9llu ", sum.user_count); @@ -558,7 +558,7 @@ timerlat_print_stats_all(struct timerlat_params *params, sum.min_thread, false); - if (params->user_hist) + if (params->user_data) format_summary_value(trace->seq, sum.user_count, sum.min_user, @@ -581,7 +581,7 @@ timerlat_print_stats_all(struct timerlat_params *params, sum.sum_thread, true); - if (params->user_hist) + if (params->user_data) format_summary_value(trace->seq, sum.user_count, sum.sum_user, @@ -604,7 +604,7 @@ timerlat_print_stats_all(struct timerlat_params *params, sum.max_thread, false); - if (params->user_hist) + if (params->user_data) format_summary_value(trace->seq, sum.user_count, sum.max_user, @@ -654,7 +654,7 @@ timerlat_print_stats(struct timerlat_params *params, struct osnoise_tool *tool) data->hist[cpu].thread[bucket]); } - if (params->user_hist) { + if (params->user_data) { total += data->hist[cpu].user[bucket]; trace_seq_printf(trace->seq, "%9d ", data->hist[cpu].user[bucket]); @@ -690,7 +690,7 @@ timerlat_print_stats(struct timerlat_params *params, struct osnoise_tool *tool) trace_seq_printf(trace->seq, "%9d ", data->hist[cpu].thread[data->entries]); - if (params->user_hist) + if (params->user_data) trace_seq_printf(trace->seq, "%9d ", data->hist[cpu].user[data->entries]); } @@ -965,7 +965,7 @@ static struct timerlat_params params->user_workload = 1; /* fallback: -u implies in -U */ case 'U': - params->user_hist = 1; + params->user_data = 1; break; case '0': /* no irq */ params->no_irq = 1; @@ -1063,98 +1063,11 @@ static struct timerlat_params static int timerlat_hist_apply_config(struct osnoise_tool *tool, struct timerlat_params *params) { - int retval, i; - - if (!params->sleep_time) - params->sleep_time = 1; - - if (params->cpus) { - retval = osnoise_set_cpus(tool->context, params->cpus); - if (retval) { - err_msg("Failed to apply CPUs config\n"); - goto out_err; - } - } else { - for (i = 0; i < sysconf(_SC_NPROCESSORS_CONF); i++) - CPU_SET(i, ¶ms->monitored_cpus); - } - - if (params->stop_us) { - retval = osnoise_set_stop_us(tool->context, params->stop_us); - if (retval) { - err_msg("Failed to set stop us\n"); - goto out_err; - } - } - - if (params->stop_total_us) { - retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us); - if (retval) { - err_msg("Failed to set stop total us\n"); - goto out_err; - } - } - - if (params->timerlat_period_us) { - retval = osnoise_set_timerlat_period_us(tool->context, params->timerlat_period_us); - if (retval) { - err_msg("Failed to set timerlat period\n"); - goto out_err; - } - } - - if (params->print_stack) { - retval = osnoise_set_print_stack(tool->context, params->print_stack); - if (retval) { - err_msg("Failed to set print stack\n"); - goto out_err; - } - } - - if (params->hk_cpus) { - retval = sched_setaffinity(getpid(), sizeof(params->hk_cpu_set), - ¶ms->hk_cpu_set); - if (retval == -1) { - err_msg("Failed to set rtla to the house keeping CPUs\n"); - goto out_err; - } - } else if (params->cpus) { - /* - * Even if the user do not set a house-keeping CPU, try to - * move rtla to a CPU set different to the one where the user - * set the workload to run. - * - * No need to check results as this is an automatic attempt. - */ - auto_house_keeping(¶ms->monitored_cpus); - } - - /* - * If the user did not specify a type of thread, try user-threads first. - * Fall back to kernel threads otherwise. - */ - if (!params->kernel_workload && !params->user_hist) { - retval = tracefs_file_exists(NULL, "osnoise/per_cpu/cpu0/timerlat_fd"); - if (retval) { - debug_msg("User-space interface detected, setting user-threads\n"); - params->user_workload = 1; - params->user_hist = 1; - } else { - debug_msg("User-space interface not detected, setting kernel-threads\n"); - params->kernel_workload = 1; - } - } + int retval; - /* - * Set workload according to type of thread if the kernel supports it. - * On kernels without support, user threads will have already failed - * on missing timerlat_fd, and kernel threads do not need it. - */ - retval = osnoise_set_workload(tool->context, params->kernel_workload); - if (retval < -1) { - err_msg("Failed to set OSNOISE_WORKLOAD option\n"); + retval = timerlat_apply_config(tool, params); + if (retval) goto out_err; - } return 0; diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index c3196a0bb5851..79cb6f28967fc 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -266,7 +266,7 @@ static void timerlat_top_header(struct timerlat_params *params, struct osnoise_t trace_seq_printf(s, "\033[2;37;40m"); trace_seq_printf(s, " Timer Latency "); - if (params->user_top) + if (params->user_data) trace_seq_printf(s, " "); if (params->pretty_output) @@ -277,7 +277,7 @@ static void timerlat_top_header(struct timerlat_params *params, struct osnoise_t params->output_divisor == 1 ? "ns" : "us", params->output_divisor == 1 ? "ns" : "us"); - if (params->user_top) { + if (params->user_data) { trace_seq_printf(s, " | Ret user Timer Latency (%s)", params->output_divisor == 1 ? "ns" : "us"); } @@ -287,7 +287,7 @@ static void timerlat_top_header(struct timerlat_params *params, struct osnoise_t trace_seq_printf(s, "\033[2;30;47m"); trace_seq_printf(s, "CPU COUNT | cur min avg max | cur min avg max"); - if (params->user_top) + if (params->user_data) trace_seq_printf(s, " | cur min avg max"); if (params->pretty_output) @@ -338,7 +338,7 @@ static void timerlat_top_print(struct osnoise_tool *top, int cpu) trace_seq_printf(s, "%9llu", cpu_data->max_thread); } - if (!params->user_top) { + if (!params->user_data) { trace_seq_printf(s, "\n"); return; } @@ -380,7 +380,7 @@ timerlat_top_print_sum(struct osnoise_tool *top, struct timerlat_top_cpu *summar } trace_seq_printf(s, "%.*s|%.*s|%.*s", 15, split, 40, split, 39, split); - if (params->user_top) + if (params->user_data) trace_seq_printf(s, "-|%.*s", 39, split); trace_seq_printf(s, "\n"); @@ -405,7 +405,7 @@ timerlat_top_print_sum(struct osnoise_tool *top, struct timerlat_top_cpu *summar trace_seq_printf(s, "%9llu", summary->max_thread); } - if (!params->user_top) { + if (!params->user_data) { trace_seq_printf(s, "\n"); return; } @@ -722,7 +722,7 @@ static struct timerlat_params params->user_workload = true; /* fallback: -u implies -U */ case 'U': - params->user_top = true; + params->user_data = true; break; case '0': /* trigger */ if (params->events) { @@ -800,100 +800,10 @@ static int timerlat_top_apply_config(struct osnoise_tool *top, struct timerlat_params *params) { int retval; - int i; - - if (!params->sleep_time) - params->sleep_time = 1; - - if (params->cpus) { - retval = osnoise_set_cpus(top->context, params->cpus); - if (retval) { - err_msg("Failed to apply CPUs config\n"); - goto out_err; - } - } else { - for (i = 0; i < sysconf(_SC_NPROCESSORS_CONF); i++) - CPU_SET(i, ¶ms->monitored_cpus); - } - - if (params->stop_us) { - retval = osnoise_set_stop_us(top->context, params->stop_us); - if (retval) { - err_msg("Failed to set stop us\n"); - goto out_err; - } - } - - if (params->stop_total_us) { - retval = osnoise_set_stop_total_us(top->context, params->stop_total_us); - if (retval) { - err_msg("Failed to set stop total us\n"); - goto out_err; - } - } - - - if (params->timerlat_period_us) { - retval = osnoise_set_timerlat_period_us(top->context, params->timerlat_period_us); - if (retval) { - err_msg("Failed to set timerlat period\n"); - goto out_err; - } - } - - if (params->print_stack) { - retval = osnoise_set_print_stack(top->context, params->print_stack); - if (retval) { - err_msg("Failed to set print stack\n"); - goto out_err; - } - } - - if (params->hk_cpus) { - retval = sched_setaffinity(getpid(), sizeof(params->hk_cpu_set), - ¶ms->hk_cpu_set); - if (retval == -1) { - err_msg("Failed to set rtla to the house keeping CPUs\n"); - goto out_err; - } - } else if (params->cpus) { - /* - * Even if the user do not set a house-keeping CPU, try to - * move rtla to a CPU set different to the one where the user - * set the workload to run. - * - * No need to check results as this is an automatic attempt. - */ - auto_house_keeping(¶ms->monitored_cpus); - } - - /* - * If the user did not specify a type of thread, try user-threads first. - * Fall back to kernel threads otherwise. - */ - if (!params->kernel_workload && !params->user_top) { - retval = tracefs_file_exists(NULL, "osnoise/per_cpu/cpu0/timerlat_fd"); - if (retval) { - debug_msg("User-space interface detected, setting user-threads\n"); - params->user_workload = 1; - params->user_top = 1; - } else { - debug_msg("User-space interface not detected, setting kernel-threads\n"); - params->kernel_workload = 1; - } - } - - /* - * Set workload according to type of thread if the kernel supports it. - * On kernels without support, user threads will have already failed - * on missing timerlat_fd, and kernel threads do not need it. - */ - retval = osnoise_set_workload(top->context, params->kernel_workload); - if (retval < -1) { - err_msg("Failed to set OSNOISE_WORKLOAD option\n"); + retval = timerlat_apply_config(top, params); + if (retval) goto out_err; - } if (isatty(STDOUT_FILENO) && !params->quiet) params->pretty_output = 1; @@ -1142,7 +1052,7 @@ int timerlat_top_main(int argc, char *argv[]) } } - if (params->cgroup && !params->user_top) { + if (params->cgroup && !params->user_data) { retval = set_comm_cgroup("timerlat/", params->cgroup_name); if (!retval) { err_msg("Failed to move threads to cgroup\n"); From a8122a63c903c0e2dd8959295eb2abfc644b36e1 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Thu, 20 Mar 2025 10:24:57 +0100 Subject: [PATCH 15/19] rtla/osnoise: Set OSNOISE_WORKLOAD to true If running rtla osnoise with NO_OSNOISE_WORKLOAD, it reports no samples: $ echo NO_OSNOISE_WORKLOAD > /sys/kernel/tracing/osnoise/options $ rtla osnoise hist -d 10s Index over: 0 count: 0 min: 0 avg: 0 max: 0 This situation can also happen when running rtla-osnoise after an improperly exited rtla-timerlat run. Set OSNOISE_WORKLOAD in rtla-osnoise, too, similarly to what we already did for timerlat in commit 217f0b1e990e ("rtla/timerlat_top: Set OSNOISE_WORKLOAD for kernel threads") and commit d8d866171a41 ("rtla/timerlat_hist: Set OSNOISE_WORKLOAD for kernel threads"). Note that there is no user workload mode for rtla-osnoise yet, so OSNOISE_WORKLOAD is always set to true. Cc: Luis Goncalves Link: https://lore.kernel.org/20250320092500.101385-4-tglozar@redhat.com Fixes: 1eceb2fc2ca5 ("rtla/osnoise: Add osnoise top mode") Fixes: 829a6c0b5698 ("rtla/osnoise: Add the hist mode") Signed-off-by: Tomas Glozar Reviewed-by: John Kacur Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/osnoise.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/tracing/rtla/src/osnoise.c b/tools/tracing/rtla/src/osnoise.c index 1735a36466c4b..a71618d876e97 100644 --- a/tools/tracing/rtla/src/osnoise.c +++ b/tools/tracing/rtla/src/osnoise.c @@ -1187,6 +1187,12 @@ osnoise_apply_config(struct osnoise_tool *tool, struct osnoise_params *params) auto_house_keeping(¶ms->monitored_cpus); } + retval = osnoise_set_workload(tool->context, true); + if (retval < -1) { + err_msg("Failed to set OSNOISE_WORKLOAD option\n"); + goto out_err; + } + return 0; out_err: From 0122938a7ab49b531b71602584da524ffebdd2f9 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Thu, 20 Mar 2025 10:24:58 +0100 Subject: [PATCH 16/19] rtla: Always set all tracer options rtla currently only sets tracer options that are explicitly set by the user, with the exception of OSNOISE_WORKLOAD. This leads to improper behavior in case rtla is run with those options not set to the default value. rtla does reset them to the original value upon exiting, but that does not protect it from starting with non-default values set either by an improperly exited rtla or by another user of the tracers. For example, after running this command: $ echo 1 > /sys/kernel/tracing/osnoise/stop_tracing_us all runs of rtla will stop at the 1us threshold, even if not requested by the user: $ rtla osnoise hist Index CPU-000 CPU-001 1 8 5 2 5 9 3 1 2 4 6 1 5 2 1 6 0 1 8 1 1 12 0 1 14 1 0 15 1 0 over: 0 0 count: 25 21 min: 1 1 avg: 3.68 3.05 max: 15 12 rtla osnoise hit stop tracing Fix the problem by setting the default value for all tracer options if the user has not provided their own value. For most of the options, it's enough to just drop the if clause checking for the value being set. For cpus, "all" is used as the default value, and for osnoise default period and runtime, default values of the osnoise_data variable in trace_osnoise.c are used. Cc: Luis Goncalves Link: https://lore.kernel.org/20250320092500.101385-5-tglozar@redhat.com Fixes: 1eceb2fc2ca5 ("rtla/osnoise: Add osnoise top mode") Fixes: 829a6c0b5698 ("rtla/osnoise: Add the hist mode") Fixes: a828cd18bc4a ("rtla: Add timerlat tool and timelart top mode") Fixes: 1eeb6328e8b3 ("rtla/timerlat: Add timerlat hist mode") Signed-off-by: Tomas Glozar Reviewed-by: John Kacur Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/src/osnoise.c | 56 ++++++++++++++--------------- tools/tracing/rtla/src/timerlat.c | 59 +++++++++++++++---------------- 2 files changed, 56 insertions(+), 59 deletions(-) diff --git a/tools/tracing/rtla/src/osnoise.c b/tools/tracing/rtla/src/osnoise.c index a71618d876e97..2dc3e4539e99b 100644 --- a/tools/tracing/rtla/src/osnoise.c +++ b/tools/tracing/rtla/src/osnoise.c @@ -17,6 +17,9 @@ #include "osnoise.h" +#define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */ +#define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */ + /* * osnoise_get_cpus - return the original "osnoise/cpus" content * @@ -1127,46 +1130,43 @@ osnoise_apply_config(struct osnoise_tool *tool, struct osnoise_params *params) if (!params->sleep_time) params->sleep_time = 1; - if (params->cpus) { - retval = osnoise_set_cpus(tool->context, params->cpus); - if (retval) { - err_msg("Failed to apply CPUs config\n"); - goto out_err; - } + retval = osnoise_set_cpus(tool->context, params->cpus ? params->cpus : "all"); + if (retval) { + err_msg("Failed to apply CPUs config\n"); + goto out_err; } if (params->runtime || params->period) { retval = osnoise_set_runtime_period(tool->context, params->runtime, params->period); - if (retval) { - err_msg("Failed to set runtime and/or period\n"); - goto out_err; - } + } else { + retval = osnoise_set_runtime_period(tool->context, + DEFAULT_SAMPLE_PERIOD, + DEFAULT_SAMPLE_RUNTIME); } - if (params->stop_us) { - retval = osnoise_set_stop_us(tool->context, params->stop_us); - if (retval) { - err_msg("Failed to set stop us\n"); - goto out_err; - } + if (retval) { + err_msg("Failed to set runtime and/or period\n"); + goto out_err; } - if (params->stop_total_us) { - retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us); - if (retval) { - err_msg("Failed to set stop total us\n"); - goto out_err; - } + retval = osnoise_set_stop_us(tool->context, params->stop_us); + if (retval) { + err_msg("Failed to set stop us\n"); + goto out_err; } - if (params->threshold) { - retval = osnoise_set_tracing_thresh(tool->context, params->threshold); - if (retval) { - err_msg("Failed to set tracing_thresh\n"); - goto out_err; - } + retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us); + if (retval) { + err_msg("Failed to set stop total us\n"); + goto out_err; + } + + retval = osnoise_set_tracing_thresh(tool->context, params->threshold); + if (retval) { + err_msg("Failed to set tracing_thresh\n"); + goto out_err; } if (params->hk_cpus) { diff --git a/tools/tracing/rtla/src/timerlat.c b/tools/tracing/rtla/src/timerlat.c index 448fb1f7d3a6a..c29e2ba2d7d8f 100644 --- a/tools/tracing/rtla/src/timerlat.c +++ b/tools/tracing/rtla/src/timerlat.c @@ -16,6 +16,8 @@ #include "timerlat.h" +#define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */ + /* * timerlat_apply_config - apply common configs to the initialized tool */ @@ -27,49 +29,44 @@ timerlat_apply_config(struct osnoise_tool *tool, struct timerlat_params *params) if (!params->sleep_time) params->sleep_time = 1; - if (params->cpus) { - retval = osnoise_set_cpus(tool->context, params->cpus); - if (retval) { - err_msg("Failed to apply CPUs config\n"); - goto out_err; - } - } else { + retval = osnoise_set_cpus(tool->context, params->cpus ? params->cpus : "all"); + if (retval) { + err_msg("Failed to apply CPUs config\n"); + goto out_err; + } + + if (!params->cpus) { for (i = 0; i < sysconf(_SC_NPROCESSORS_CONF); i++) CPU_SET(i, ¶ms->monitored_cpus); } - if (params->stop_us) { - retval = osnoise_set_stop_us(tool->context, params->stop_us); - if (retval) { - err_msg("Failed to set stop us\n"); - goto out_err; - } + retval = osnoise_set_stop_us(tool->context, params->stop_us); + if (retval) { + err_msg("Failed to set stop us\n"); + goto out_err; } - if (params->stop_total_us) { - retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us); - if (retval) { - err_msg("Failed to set stop total us\n"); - goto out_err; - } + retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us); + if (retval) { + err_msg("Failed to set stop total us\n"); + goto out_err; } - if (params->timerlat_period_us) { - retval = osnoise_set_timerlat_period_us(tool->context, params->timerlat_period_us); - if (retval) { - err_msg("Failed to set timerlat period\n"); - goto out_err; - } + retval = osnoise_set_timerlat_period_us(tool->context, + params->timerlat_period_us ? + params->timerlat_period_us : + DEFAULT_TIMERLAT_PERIOD); + if (retval) { + err_msg("Failed to set timerlat period\n"); + goto out_err; } - if (params->print_stack) { - retval = osnoise_set_print_stack(tool->context, params->print_stack); - if (retval) { - err_msg("Failed to set print stack\n"); - goto out_err; - } + retval = osnoise_set_print_stack(tool->context, params->print_stack); + if (retval) { + err_msg("Failed to set print stack\n"); + goto out_err; } if (params->hk_cpus) { From 6c6182728a9bb02b0a973775869c9a0ceddd9658 Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Thu, 20 Mar 2025 10:24:59 +0100 Subject: [PATCH 17/19] rtla/tests: Reset osnoise options before check Remove any dangling tracing instances from previous improperly exited runs of rtla, and reset osnoise options to default before running a test case. This ensures that the test results are deterministic. Specific test cases checked that rtla behaves correctly even when the tracer state is not clean will be added later. Cc: John Kacur Cc: Luis Goncalves Link: https://lore.kernel.org/20250320092500.101385-6-tglozar@redhat.com Signed-off-by: Tomas Glozar Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/tests/engine.sh | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tools/tracing/rtla/tests/engine.sh b/tools/tracing/rtla/tests/engine.sh index 64d0446dc28e5..5db8aa4bc0319 100644 --- a/tools/tracing/rtla/tests/engine.sh +++ b/tools/tracing/rtla/tests/engine.sh @@ -8,12 +8,44 @@ test_begin() { [ -n "$TEST_COUNT" ] && echo "1..$TEST_COUNT" } +reset_osnoise() { + # Reset osnoise options to default and remove any dangling instances created + # by improperly exited rtla runs. + pushd /sys/kernel/tracing || return 1 + + # Remove dangling instances created by previous rtla run + echo 0 > tracing_thresh + cd instances + for i in osnoise_top osnoise_hist timerlat_top timerlat_hist + do + [ ! -d "$i" ] && continue + rmdir "$i" + done + + # Reset options to default + # Note: those are copied from the default values of osnoise_data + # in kernel/trace/trace_osnoise.c + cd ../osnoise + echo all > cpus + echo DEFAULTS > options + echo 1000000 > period_us + echo 0 > print_stack + echo 1000000 > runtime_us + echo 0 > stop_tracing_total_us + echo 0 > stop_tracing_us + echo 1000 > timerlat_period_us + + popd +} + check() { # Simple check: run rtla with given arguments and test exit code. # If TEST_COUNT is set, run the test. Otherwise, just count. ctr=$(($ctr + 1)) if [ -n "$TEST_COUNT" ] then + # Reset osnoise options before running test. + [ "$NO_RESET_OSNOISE" == 1 ] || reset_osnoise # Run rtla; in case of failure, include its output as comment # in the test results. result=$(stdbuf -oL $TIMEOUT "$RTLA" $2 2>&1); exitcode=$? @@ -37,6 +69,14 @@ unset_timeout() { unset TIMEOUT } +set_no_reset_osnoise() { + NO_RESET_OSNOISE=1 +} + +unset_no_reset_osnoise() { + unset NO_RESET_OSNOISE +} + test_end() { # If running without TEST_COUNT, tests are not actually run, just # counted. In that case, re-run the test with the correct count. From a86150f310d7c986bb27c3633520336b67388afe Mon Sep 17 00:00:00 2001 From: Tomas Glozar Date: Thu, 20 Mar 2025 10:25:00 +0100 Subject: [PATCH 18/19] rtla/tests: Test setting default options Add function to test engine to test with pre-set osnoise options, and use it to test whether osnoise period (as an example) is set correctly. The test works by pre-setting a high period of 10 minutes and stop on threshold. Thus, it is easy to check whether rtla is properly resetting the period to default: if it is, the test will complete on time, since the first sample will overflow the threshold. If not, it will time out. Cc: Luis Goncalves Link: https://lore.kernel.org/20250320092500.101385-7-tglozar@redhat.com Signed-off-by: Tomas Glozar Reviewed-by: John Kacur Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/tests/engine.sh | 26 ++++++++++++++++++++++++++ tools/tracing/rtla/tests/osnoise.t | 6 ++++++ 2 files changed, 32 insertions(+) diff --git a/tools/tracing/rtla/tests/engine.sh b/tools/tracing/rtla/tests/engine.sh index 5db8aa4bc0319..b1697b3e3f528 100644 --- a/tools/tracing/rtla/tests/engine.sh +++ b/tools/tracing/rtla/tests/engine.sh @@ -61,6 +61,32 @@ check() { fi } +check_with_osnoise_options() { + # Do the same as "check", but with pre-set osnoise options. + # Note: rtla should reset the osnoise options, this is used to test + # if it indeed does so. + # Save original arguments + arg1=$1 + arg2=$2 + + # Apply osnoise options (if not dry run) + if [ -n "$TEST_COUNT" ] + then + [ "$NO_RESET_OSNOISE" == 1 ] || reset_osnoise + shift + while shift + do + [ "$1" == "" ] && continue + option=$(echo $1 | cut -d '=' -f 1) + value=$(echo $1 | cut -d '=' -f 2) + echo "option: $option, value: $value" + echo "$value" > "/sys/kernel/tracing/osnoise/$option" || return 1 + done + fi + + NO_RESET_OSNOISE=1 check "$arg1" "$arg2" +} + set_timeout() { TIMEOUT="timeout -v -k 15s $1" } diff --git a/tools/tracing/rtla/tests/osnoise.t b/tools/tracing/rtla/tests/osnoise.t index 86596e5478939..e5995c03c790f 100644 --- a/tools/tracing/rtla/tests/osnoise.t +++ b/tools/tracing/rtla/tests/osnoise.t @@ -16,4 +16,10 @@ check "verify the --trace param" \ check "verify the --entries/-E param" \ "osnoise hist -P F:1 -c 0 -r 900000 -d 1M -b 10 -E 25" +# Test setting default period by putting an absurdly high period +# and stopping on threshold. +# If default period is not set, this will time out. +check_with_osnoise_options "apply default period" \ + "osnoise hist -s 1" period_us=600000000 + test_end From 732032692f6ae311bc35159b18e5b7c5e64010fc Mon Sep 17 00:00:00 2001 From: John Kacur Date: Fri, 21 Mar 2025 13:50:53 -0400 Subject: [PATCH 19/19] rtla: Add the ability to create ctags and etags - Add the ability to create and remove ctags and etags, using the following make tags make TAGS make tags_clean - fix a comment in Makefile.rtla with the correct spelling and don't imply that the ability to create an rtla tarball will be removed Cc: Tomas Glozar Cc: "Luis Claudio R . Goncalves" Link: https://lore.kernel.org/20250321175053.29048-1-jkacur@redhat.com Signed-off-by: John Kacur Signed-off-by: Steven Rostedt (Google) --- tools/tracing/rtla/Makefile.rtla | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/tracing/rtla/Makefile.rtla b/tools/tracing/rtla/Makefile.rtla index cc1d6b615475f..08c1b40883d3a 100644 --- a/tools/tracing/rtla/Makefile.rtla +++ b/tools/tracing/rtla/Makefile.rtla @@ -34,6 +34,8 @@ INSTALL := install MKDIR := mkdir STRIP := strip BINDIR := /usr/bin +CTAGS := ctags +ETAGS := ctags -e .PHONY: install install: doc_install @@ -47,6 +49,18 @@ install: doc_install @test ! -f $(DESTDIR)$(BINDIR)/timerlat || $(RM) $(DESTDIR)$(BINDIR)/timerlat @$(LN) -s rtla $(DESTDIR)$(BINDIR)/timerlat +.PHONY: tags +tags: + $(CTAGS) -R --extras=+f --c-kinds=+p src + +.PHONY: TAGS +TAGS: + $(ETAGS) -R --extras=+f --c-kinds=+p src + +.PHONY: tags_clean +tags_clean: + $(RM) tags TAGS + .PHONY: doc doc_clean doc_install doc: $(MAKE) -C $(DOCSRC) @@ -57,8 +71,7 @@ doc_clean: doc_install: $(MAKE) -C $(DOCSRC) install -# This section is neesary for the tarball, when the tarball -# support is removed, we can delete these entries. +# This section is necessary to make the rtla tarball NAME := rtla DIRS := src FILES := Makefile README.txt