diff --git a/include/nvtop/interface_internal_common.h b/include/nvtop/interface_internal_common.h index aec93d3..5e49072 100644 --- a/include/nvtop/interface_internal_common.h +++ b/include/nvtop/interface_internal_common.h @@ -140,6 +140,16 @@ struct nvtop_interface { unsigned num_plots; struct plot_window *plots; interface_ring_buffer saved_data_ring; + // PCIe throughput history, kept separately from saved_data_ring so it is not + // subject to the MAX_LINES_PER_PLOT budget. Slot 0 is rx, slot 1 is tx, both + // in KB/s as reported by the vendor backend. + interface_ring_buffer pcie_ring; + // Scale of the PCIe overlay: the generation and width of the negotiated link. + // The per-sample link the backends report trains down to Gen1 while the GPU is + // idle, so these keep the maximum instead. See save_current_data_to_ring for + // where each comes from. + unsigned *pcie_max_gen; + unsigned *pcie_max_width; struct setup_window setup_win; }; diff --git a/include/nvtop/interface_options.h b/include/nvtop/interface_options.h index 53324ae..55a22b9 100644 --- a/include/nvtop/interface_options.h +++ b/include/nvtop/interface_options.h @@ -52,6 +52,7 @@ typedef struct nvtop_interface_option_struct { bool has_monitored_set_changed; // True if the set of monitored gpu was modified through the interface bool has_gpu_info_bar; // Show info bar with additional GPU parameters bool hide_processes_list; // Hide processes list + bool show_pcie_overlay; // Shade the charts with the PCIe rx/tx link utilization unsigned char gpu_plot_color_idx[MAX_LINES_PER_PLOT]; // index into plot_color_names[] per plot slot } nvtop_interface_option; diff --git a/include/nvtop/plot.h b/include/nvtop/plot.h index 65bf621..a8c6cbb 100644 --- a/include/nvtop/plot.h +++ b/include/nvtop/plot.h @@ -30,9 +30,23 @@ #define PLOT_MAX_LEGEND_SIZE 35 +// PCIe directions tracked for the bandwidth overlay: rx first, then tx. +#define PCIE_DIRECTION_COUNT 2 + void nvtop_line_plot(WINDOW *win, size_t num_data, const double *data, unsigned num_plots, bool legend_left, char legend[MAX_LINES_PER_PLOT][PLOT_MAX_LEGEND_SIZE]); +// Theoretical unidirectional payload bandwidth of a PCIe link, in KB/s. +// Returns 0 when the generation or width is unknown. +unsigned nvtop_pcie_link_max_kbs(unsigned gen, unsigned width); + +// Shade the background of an already drawn plot from the bottom of each column +// up to the given utilization. rx_fraction[] and tx_fraction[] hold one value in +// [0,1] per terminal column. Only cell attributes are touched, so the lines and +// legend drawn by nvtop_line_plot stay where they are. +void nvtop_bandwidth_overlay(WINDOW *win, size_t num_data, const double *rx_fraction, const double *tx_fraction, + short rx_color, short tx_color); + void draw_rectangle(WINDOW *win, unsigned startX, unsigned startY, unsigned sizeX, unsigned sizeY); #endif // __PLOT_H_ diff --git a/src/interface.c b/src/interface.c index ae23199..99e09dc 100644 --- a/src/interface.c +++ b/src/interface.c @@ -466,6 +466,17 @@ struct nvtop_interface *initialize_curses(unsigned total_devices, unsigned devic } interface_alloc_ring_buffer(devices_count, 4, 10 * 60 * 1000, &interface->saved_data_ring); + interface_alloc_ring_buffer(devices_count, PCIE_DIRECTION_COUNT, 10 * 60 * 1000, &interface->pcie_ring); + interface->pcie_max_gen = calloc(devices_count, sizeof(*interface->pcie_max_gen)); + if (!interface->pcie_max_gen) { + perror("Cannot allocate memory: "); + exit(EXIT_FAILURE); + } + interface->pcie_max_width = calloc(devices_count, sizeof(*interface->pcie_max_width)); + if (!interface->pcie_max_width) { + perror("Cannot allocate memory: "); + exit(EXIT_FAILURE); + } initialize_all_windows(interface); return interface; } @@ -481,6 +492,9 @@ void clean_ncurses(struct nvtop_interface *interface) { free(interface->options.config_file_location); free(interface->devices_win); interface_free_ring_buffer(&interface->saved_data_ring); + interface_free_ring_buffer(&interface->pcie_ring); + free(interface->pcie_max_gen); + free(interface->pcie_max_width); free(interface); } @@ -1744,6 +1758,31 @@ void save_current_data_to_ring(struct list_head *devices, struct nvtop_interface } } + // PCIe throughput is recorded unconditionally: it is an overlay, not one of + // the user-selectable plot lines. + unsigned pcie_rx = 0, pcie_tx = 0; + if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, pcie_rx)) + pcie_rx = device->dynamic_info.pcie_rx; + if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, pcie_tx)) + pcie_tx = device->dynamic_info.pcie_tx; + interface_ring_buffer_push(&interface->pcie_ring, dev_id, 0, pcie_rx); + interface_ring_buffer_push(&interface->pcie_ring, dev_id, 1, pcie_tx); + // Static max_pcie_gen/max_pcie_link_width report device or slot capability, not the + // negotiated link, so scaling tracks the highest negotiated (dynamic) value seen + // instead. The static value is only used as a fallback for a backend that never + // reports a dynamic reading. + if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, pcie_link_gen) && + device->dynamic_info.pcie_link_gen > interface->pcie_max_gen[dev_id]) + interface->pcie_max_gen[dev_id] = device->dynamic_info.pcie_link_gen; + else if (interface->pcie_max_gen[dev_id] == 0 && GPUINFO_STATIC_FIELD_VALID(&device->static_info, max_pcie_gen)) + interface->pcie_max_gen[dev_id] = device->static_info.max_pcie_gen; + if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, pcie_link_width) && + device->dynamic_info.pcie_link_width > interface->pcie_max_width[dev_id]) + interface->pcie_max_width[dev_id] = device->dynamic_info.pcie_link_width; + else if (interface->pcie_max_width[dev_id] == 0 && + GPUINFO_STATIC_FIELD_VALID(&device->static_info, max_pcie_link_width)) + interface->pcie_max_width[dev_id] = device->static_info.max_pcie_link_width; + dev_id++; } } @@ -1829,6 +1868,82 @@ static unsigned populate_plot_data_from_ring_buffer(const struct nvtop_interface return total_to_draw; } +// Fill one fraction-per-column array from the PCIe ring buffer, replicating each +// sample across columns_per_sample columns so the bars line up in time with the +// line plot above them. Returns the most recent (current) sample's fraction, the +// same instant the top-of-screen RX/TX readout and the rest of nvtop's live +// percentages reflect: a mean over the whole visible window would stay low while +// a spike confined to the last few samples is still clearly visible in the bars. +static double populate_pcie_bar_data(const struct nvtop_interface *interface, unsigned dev_id, unsigned which, + unsigned link_kbs, unsigned columns_per_sample, size_t num_columns, + double *fraction) { + memset(fraction, 0, num_columns * sizeof(*fraction)); + unsigned max_samples = num_columns / columns_per_sample; + unsigned stored = interface_ring_buffer_data_stored(&interface->pcie_ring, dev_id, which); + double current = 0.; + + for (unsigned j = 0; j < stored && j < max_samples; ++j) { + unsigned value = interface_ring_buffer_get(&interface->pcie_ring, dev_id, which, stored - j - 1); + double f = (double)value / (double)link_kbs; + unsigned slot = interface->options.plot_left_to_right ? j : max_samples - j - 1; + for (unsigned c = 0; c < columns_per_sample; ++c) { + size_t col = (size_t)slot * columns_per_sample + c; + if (col < num_columns) + fraction[col] = f; + } + if (j == 0) + current = f > 1. ? 1. : f; + } + return current; +} + +// Shade the plot background with the rx/tx link utilization, scaled to the +// fastest link the device has been seen running at. The overlay tracks the first +// device drawn in this window; extend the loop if you want one per device. +static void draw_pcie_overlay(const struct nvtop_interface *interface, const struct plot_window *plot, + unsigned num_lines, unsigned plot_rows, unsigned plot_cols, bool legend_left) { + if (!interface->options.show_pcie_overlay || plot->num_devices_to_plot == 0 || num_lines == 0) + return; + unsigned dev_id = plot->devices_ids[0]; + unsigned gen = interface->pcie_max_gen[dev_id]; + unsigned width = interface->pcie_max_width[dev_id]; + unsigned link_kbs = nvtop_pcie_link_max_kbs(gen, width); + if (link_kbs == 0) + return; + + size_t num_columns = plot->num_data; + double *fraction = malloc(PCIE_DIRECTION_COUNT * num_columns * sizeof(*fraction)); + if (!fraction) + return; + + // Ring slot 0 is rx, slot 1 is tx. + static const short direction_color[PCIE_DIRECTION_COUNT] = {green_color, magenta_color}; + static const char *const direction_name[PCIE_DIRECTION_COUNT] = {"rx", "tx"}; + double current[PCIE_DIRECTION_COUNT]; + for (unsigned which = 0; which < PCIE_DIRECTION_COUNT; ++which) + current[which] = populate_pcie_bar_data(interface, dev_id, which, link_kbs, num_lines, num_columns, + fraction + which * num_columns); + nvtop_bandwidth_overlay(plot->plot_window, num_columns, fraction, fraction + num_columns, direction_color[0], + direction_color[1]); + free(fraction); + + // Continue the plot legend in the shading colors, so each color still names + // what it stands for. The percentage is the current utilization of the link, + // the same instant the top-of-screen RX/TX readout reflects. + for (unsigned which = 0; which < PCIE_DIRECTION_COUNT; ++which) { + unsigned row = num_lines + which; + if (row >= plot_rows) + break; + char legend[PLOT_MAX_LEGEND_SIZE]; + int len = snprintf(legend, sizeof(legend), "GPU%u pcie %s%3.0f%%", dev_id, direction_name[which], + current[which] * 100.); + if (len <= 0 || len > (int)plot_cols) + continue; + wcolor_set(plot->plot_window, direction_color[which], NULL); + mvwprintw(plot->plot_window, row, legend_left ? 0 : (int)plot_cols - len, "%s", legend); + } +} + static void draw_plots(struct nvtop_interface *interface) { for (unsigned plot_id = 0; plot_id < interface->num_plots; ++plot_id) { werase(interface->plots[plot_id].plot_window); @@ -1839,8 +1954,14 @@ static void draw_plots(struct nvtop_interface *interface) { populate_plot_data_from_ring_buffer(interface, &interface->plots[plot_id], interface->plots[plot_id].num_data, interface->plots[plot_id].data, plot_legend); + int plot_rows, plot_cols; + getmaxyx(interface->plots[plot_id].plot_window, plot_rows, plot_cols); + bool legend_left = !interface->options.plot_left_to_right; + nvtop_line_plot(interface->plots[plot_id].plot_window, interface->plots[plot_id].num_data, - interface->plots[plot_id].data, num_lines, !interface->options.plot_left_to_right, plot_legend); + interface->plots[plot_id].data, num_lines, legend_left, plot_legend); + + draw_pcie_overlay(interface, &interface->plots[plot_id], num_lines, plot_rows, plot_cols, legend_left); wnoutrefresh(interface->plots[plot_id].plot_window); } diff --git a/src/interface_options.c b/src/interface_options.c index c482f3e..cee9706 100644 --- a/src/interface_options.c +++ b/src/interface_options.c @@ -129,6 +129,7 @@ void alloc_interface_options_internals(char *config_location, unsigned num_devic options->show_startup_messages = true; options->filter_nvtop_pid = true; options->has_gpu_info_bar = false; + options->show_pcie_overlay = false; options->gpu_plot_color_idx[0] = 1; // Cyan options->gpu_plot_color_idx[1] = 3; // Yellow options->gpu_plot_color_idx[2] = 2; // Green @@ -178,6 +179,7 @@ static const char header_value_gpu_info_bar[] = "GPUInfoBar"; static const char chart_section[] = "ChartOption"; static const char chart_value_reverse[] = "ReverseChart"; +static const char chart_value_pcie_overlay[] = "PcieOverlay"; static const char *chart_value_gpu_plot_color[MAX_LINES_PER_PLOT] = { "GpuPlotColor0", "GpuPlotColor1", "GpuPlotColor2", "GpuPlotColor3"}; @@ -264,6 +266,14 @@ static int nvtop_option_ini_handler(void *user, const char *section, const char ini_data->options->plot_left_to_right = false; } } + if (strcmp(name, chart_value_pcie_overlay) == 0) { + if (strcmp(value, "true") == 0) { + ini_data->options->show_pcie_overlay = true; + } + if (strcmp(value, "false") == 0) { + ini_data->options->show_pcie_overlay = false; + } + } for (unsigned s = 0; s < MAX_LINES_PER_PLOT; ++s) { if (strcmp(name, chart_value_gpu_plot_color[s]) == 0) { for (unsigned i = 0; i < plot_color_names_count; ++i) { @@ -422,6 +432,7 @@ bool save_interface_options_to_config_file(unsigned total_dev_count, const nvtop // Chart Options fprintf(config_file, "\n[%s]\n", chart_section); fprintf(config_file, "%s = %s\n", chart_value_reverse, boolean_string(options->plot_left_to_right)); + fprintf(config_file, "%s = %s\n", chart_value_pcie_overlay, boolean_string(options->show_pcie_overlay)); for (unsigned s = 0; s < MAX_LINES_PER_PLOT; ++s) fprintf(config_file, "%s = %s\n", chart_value_gpu_plot_color[s], plot_color_names[options->gpu_plot_color_idx[s]]); diff --git a/src/interface_setup_win.c b/src/interface_setup_win.c index 27d685e..04d4325 100644 --- a/src/interface_setup_win.c +++ b/src/interface_setup_win.c @@ -69,12 +69,14 @@ static const char *setup_header_option_descriptions[setup_header_options_count] enum setup_chart_options { setup_chart_reverse, + setup_chart_pcie_overlay, setup_chart_color_start, // dynamic color rows: slots 0..slot_count-1 // setup_chart_all_gpu = setup_chart_color_start + slot_count (computed) // setup_chart_start_gpu_list = setup_chart_color_start + slot_count+1 (computed) }; static const char *setup_chart_reverse_description = "Reverse plot direction"; +static const char *setup_chart_pcie_overlay_description = "Shade charts with PCIe rx/tx utilization"; static const char *setup_chart_all_gpu_description = "Displayed all GPUs"; static const char *setup_chart_gpu_description = "Displayed GPU"; @@ -376,6 +378,15 @@ static void draw_setup_window_chart(unsigned devices_count, struct list_head *de mvwchgat(option_list_win, setup_chart_reverse + 1, 0, 3, A_STANDOUT, cyan_color, NULL); } + // PCIe bandwidth overlay + option_state = interface->options.show_pcie_overlay; + mvwprintw(option_list_win, setup_chart_pcie_overlay + 1, 0, "[%c] %s", option_state_char(option_state), + setup_chart_pcie_overlay_description); + if (interface->setup_win.indentation_level == 1 && + interface->setup_win.options_selected[0] == setup_chart_pcie_overlay) { + mvwchgat(option_list_win, setup_chart_pcie_overlay + 1, 0, 3, A_STANDOUT, cyan_color, NULL); + } + // Dynamic color rows — one per active plot slot // Build slot labels from GPU 0's active metrics (representative) const char *slot_labels[MAX_LINES_PER_PLOT]; @@ -834,6 +845,9 @@ void handle_setup_win_keypress(int keyId, struct nvtop_interface *interface) { if (interface->setup_win.options_selected[0] == setup_chart_reverse) { interface->options.plot_left_to_right = !interface->options.plot_left_to_right; } + if (interface->setup_win.options_selected[0] == setup_chart_pcie_overlay) { + interface->options.show_pcie_overlay = !interface->options.show_pcie_overlay; + } // Color rows unsigned sel = interface->setup_win.options_selected[0]; if (sel >= setup_chart_color_start && sel < chart_all_gpu_kp) { diff --git a/src/plot.c b/src/plot.c index 44f2b37..cc77eb9 100644 --- a/src/plot.c +++ b/src/plot.c @@ -123,6 +123,75 @@ void nvtop_line_plot(WINDOW *win, size_t num_data, const double *data, unsigned } } +// Per-lane unidirectional payload bandwidth in KB/s, indexed by PCIe generation. +// Gen1/2 are 8b/10b encoded, Gen3 to Gen5 are 128b/130b, Gen6 is PAM4 with FLIT +// mode. These are the raw link rates minus encoding overhead; real payload +// throughput also loses a few percent to TLP headers. +static const unsigned pcie_lane_kbs_per_gen[] = { + 0, // unknown + 250000, // Gen1 2.5 GT/s + 500000, // Gen2 5 GT/s + 984615, // Gen3 8 GT/s + 1969231, // Gen4 16 GT/s + 3938461, // Gen5 32 GT/s + 7563000, // Gen6 64 GT/s +}; + +unsigned nvtop_pcie_link_max_kbs(unsigned gen, unsigned width) { + static const unsigned num_gens = sizeof(pcie_lane_kbs_per_gen) / sizeof(*pcie_lane_kbs_per_gen); + if (gen == 0 || width == 0) + return 0; + if (gen >= num_gens) + gen = num_gens - 1; + return pcie_lane_kbs_per_gen[gen] * width; +} + +// Height in cells of a bar covering the given fraction of a plot_rows tall plot. +// Traffic worth less than half a cell shades nothing, so an idle link reads as +// idle instead of as a permanent one row floor. +static unsigned bar_cells(double fraction, int plot_rows) { + if (!(fraction > 0.)) + return 0; + if (fraction > 1.) + fraction = 1.; + double cells = round(fraction * (double)plot_rows); + return (unsigned)cells; +} + +void nvtop_bandwidth_overlay(WINDOW *win, size_t num_data, const double *rx_fraction, const double *tx_fraction, + short rx_color, short tx_color) { + int rows, cols; + getmaxyx(win, rows, cols); + // nvtop_line_plot places 100% at row 0 and 0% at row rows-1, using the full + // window height. A bar covering the whole height (plot_rows == rows) is + // needed to reach row 0 at 100%, matching that gridline. + int plot_rows = rows; + if (plot_rows < 1) + return; + + for (size_t i = 0; i < num_data && i < (size_t)cols; ++i) { + const double fraction[PCIE_DIRECTION_COUNT] = {rx_fraction[i], tx_fraction[i]}; + const short color[PCIE_DIRECTION_COUNT] = {rx_color, tx_color}; + // Both directions rise from the bottom of the plot, so paint the taller one + // first: the shorter one then reads as a band inside it and neither height + // is lost. + unsigned tallest = fraction[0] >= fraction[1] ? 0 : 1; + for (unsigned o = 0; o < PCIE_DIRECTION_COUNT; ++o) { + unsigned which = o == 0 ? tallest : PCIE_DIRECTION_COUNT - 1 - tallest; + unsigned cells = bar_cells(fraction[which], plot_rows); + for (unsigned r = 0; r < cells; ++r) { + int row = rows - 1 - (int)r; + // Reverse video turns the cell background into the pair's color while + // leaving whatever the line plot drew there legible on top of it. The + // cell's own attributes have to be carried over: dropping A_ALTCHARSET + // would turn the line drawing glyphs back into the ASCII they map to. + chtype existing = mvwinch(win, row, i) & A_ATTRIBUTES & ~A_COLOR; + mvwchgat(win, row, i, 1, existing | A_REVERSE, color[which], NULL); + } + } + } +} + void draw_rectangle(WINDOW *win, unsigned startX, unsigned startY, unsigned sizeX, unsigned sizeY) { mvwhline(win, startY, startX + 1, 0, sizeX - 2); mvwhline(win, startY + sizeY - 1, startX + 1, 0, sizeX - 2);