From 895eb6b36c72e1eec007b12d07b67a84836a9473 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 27 Jan 2016 21:08:59 +0100 Subject: [PATCH] Add the new --test-io-stats to show graphs, and reformat them --- HISTORY | 4 + cmdline/snapraid.c | 9 +- cmdline/state.c | 330 +++++++++++++++++++++++---------------------- cmdline/state.h | 6 +- 4 files changed, 180 insertions(+), 169 deletions(-) diff --git a/HISTORY b/HISTORY index 59ae285a..5237aa80 100644 --- a/HISTORY +++ b/HISTORY @@ -11,6 +11,10 @@ SnapRAID HISTORY You can control the number of blocks to cache with the option --test-io-cache=NUMBER, where the number is between 3 and 128. The default is 8 MiB of blocks. + You can show runtime stats during the processing with the + --test-io-stats option. You will see a graph with the number of + cached blocks (higher is better), and a graph with the wait time + percentage (lower is better) for all the disks and computations. * The -h, --pre-hash command, saves the content file only after having verified all the hashes. This allows recovering of moved files in case a silent error is found during the hash verification diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index c78cc003..ce6332cc 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -251,6 +251,7 @@ void config(char* conf, size_t conf_size, const char* argv0) #define OPT_TEST_MATCH_FIRST_UUID 288 #define OPT_TEST_FORCE_PARITY_UPDATE 289 #define OPT_TEST_IO_CACHE 290 +#define OPT_TEST_IO_STATS 291 #if HAVE_GETOPT_LONG struct option long_options[] = { @@ -378,9 +379,12 @@ struct option long_options[] = { /* Force parity update even if all the data is not changed */ { "test-force-parity-update", 0, 0, OPT_TEST_FORCE_PARITY_UPDATE }, - /* Number of read-ahead buffers */ + /* Number of IO buffers */ { "test-io-cache", 1, 0, OPT_TEST_IO_CACHE }, + /* Print IO stats */ + { "test-io-stats", 0, 0, OPT_TEST_IO_STATS }, + { 0, 0, 0, 0 } }; #endif @@ -770,6 +774,9 @@ int main(int argc, char* argv[]) /* LCOV_EXCL_STOP */ } break; + case OPT_TEST_IO_STATS : + opt.force_stats = 1; + break; default : /* LCOV_EXCL_START */ log_fatal("Unknown option '%c'\n", (char)c); diff --git a/cmdline/state.c b/cmdline/state.c index a93dfa53..3e9b568d 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -3908,12 +3908,169 @@ void state_progress_restart(struct snapraid_state* state) #define PROGRESS_CLEAR " " +/** + * Set the latest tick data in the progress vector. + */ +static void state_progress_latest(struct snapraid_state* state) +{ + tommy_node* i; + unsigned l; + + state->progress_tick_misc[state->progress_ptr] = state->tick_misc; + state->progress_tick_raid[state->progress_ptr] = state->tick_raid; + state->progress_tick_hash[state->progress_ptr] = state->tick_hash; + state->progress_tick_io[state->progress_ptr] = state->tick_io; + for (i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + disk->progress_tick[state->progress_ptr] = disk->tick; + } + for (l = 0; l < state->level; ++l) + state->parity[l].progress_tick[state->progress_ptr] = state->parity[l].tick; +} + +/** + * Number of columns + */ +#define GRAPH_COLUMN 68 + +/** + * Get the reference value, 0 if index is PROGRESS_MAX + */ +#define ref(map,index) (index < PROGRESS_MAX ? map[index] : 0) + +static void state_progress_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest) +{ + uint64_t v; + uint64_t tick_total; + unsigned bar; + tommy_node* i; + unsigned l; + size_t pad; + + tick_total = 0; + + tick_total += state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest); + tick_total += state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest); + tick_total += state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest); + tick_total += state->progress_tick_io[current] - ref(state->progress_tick_io, oldest); + if (!tick_total) + return; + + pad = 4; + for (i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + size_t len = strlen(disk->name); + if (pad < len) + pad = len; + } + for (l = 0; l < state->level; ++l) { + size_t len = strlen(lev_config_name(l)); + if (pad < len) + pad = len; + } + + /* extra space */ + pad += 1; + + if (pad + 30 < GRAPH_COLUMN) + bar = GRAPH_COLUMN - pad; + else + bar = 30; + + if (io) { + const char* legend = "cached blocks (instant, higher is better)"; + + /* refresh the cached blocks info */ + io_refresh(io); + + printf("\n"); + + for (i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + v = disk->cached; + printr(disk->name, pad); + printf("%4" PRIu64 " | ", v); + printc('o', v * bar / io->io_max); + printf("\n"); + } + + for (l = 0; l < state->level; ++l) { + v = state->parity[l].cached; + printr(lev_config_name(l), pad); + printf("%4" PRIu64 " | ", v); + printc('o', v * bar / io->io_max); + printf("\n"); + } + + printr("", pad); + printf(" |_"); + printc('_', bar); + printf("\n"); + + printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2); + printf(legend); + printf("\n"); + } + + printf("\n"); + + for (i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + v = disk->progress_tick[current] - ref(disk->progress_tick, oldest); + printr(disk->name, pad); + printf("%3" PRIu64 "%% | ", v * 100 / tick_total); + printc('*', v * bar / tick_total); + printf("\n"); + } + + for (l = 0; l < state->level; ++l) { + v = state->parity[l].progress_tick[current] - ref(state->parity[l].progress_tick, oldest); + printr(lev_config_name(l), pad); + printf("%3" PRIu64 "%% | ", v * 100 / tick_total); + printc('*', v * bar / tick_total); + printf("\n"); + } + + v = state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest); + printr("raid", pad); + printf("%3" PRIu64 "%% | ", v * 100 / tick_total); + printc('*', v * bar / tick_total); + printf("\n"); + + v = state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest); + printr("hash", pad); + printf("%3" PRIu64 "%% | ", v * 100 / tick_total); + printc('*', v * bar / tick_total); + printf("\n"); + + v = state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest); + printr("misc", pad); + printf("%3" PRIu64 "%% | ", v * 100 / tick_total); + printc('*', v * bar / tick_total); + printf("\n"); + + printr("", pad); + printf(" |_"); + printc('_', bar); + printf("\n"); + + if (oldest == PROGRESS_MAX) { + const char* legend = "wait time"; + printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2); + printf(legend); + printf("\n"); + } else { + const char* legend = "wait time (%d secs, lower is better)"; + printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2); + printf(legend, PROGRESS_MAX); + printf("\n"); + } +} + int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize) { time_t now; int pred; - tommy_node* i; - unsigned l; now = time(0); @@ -3936,16 +4093,7 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o state->progress_time[state->progress_ptr] = now; state->progress_pos[state->progress_ptr] = countpos; state->progress_size[state->progress_ptr] = countsize; - state->progress_tick_misc[state->progress_ptr] = state->tick_misc; - state->progress_tick_raid[state->progress_ptr] = state->tick_raid; - state->progress_tick_hash[state->progress_ptr] = state->tick_hash; - state->progress_tick_io[state->progress_ptr] = state->tick_io; - for (i = state->disklist; i != 0; i = i->next) { - struct snapraid_disk* disk = i->data; - disk->progress_tick[state->progress_ptr] = disk->tick; - } - for (l = 0; l < state->level; ++l) - state->parity[l].progress_tick[state->progress_ptr] = state->parity[l].tick; + state_progress_latest(state); elapsed = now - state->progress_whole_start - state->progress_wasted; @@ -4017,9 +4165,9 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o if (delta_pos != 0) out_eta = (countmax - countpos) * delta_time / (60 * delta_pos); - if (msg_level >= MSG_VERBOSE) { + if (state->opt.force_stats) { os_clear(); - state_usage_graph(state, io, state->progress_ptr, oldest); + state_progress_graph(state, io, state->progress_ptr, oldest); } } @@ -4136,160 +4284,16 @@ void state_usage_parity(struct snapraid_state* state, unsigned* waiting_map, uns state->tick_last = now; } -#define USAGE_COLUMN 60 - -void state_usage_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest) -{ - uint64_t v; - uint64_t tick_total; - tommy_node* i; - unsigned l; - size_t pad; - - tick_total = 0; - - tick_total += state->progress_tick_misc[current] - state->progress_tick_misc[oldest]; - tick_total += state->progress_tick_raid[current] - state->progress_tick_raid[oldest]; - tick_total += state->progress_tick_hash[current] - state->progress_tick_hash[oldest]; - tick_total += state->progress_tick_io[current] - state->progress_tick_io[oldest]; - if (!tick_total) - return; - - pad = 4; - for (i = state->disklist; i != 0; i = i->next) { - struct snapraid_disk* disk = i->data; - size_t len = strlen(disk->name); - if (pad < len) - pad = len; - } - for (l = 0; l < state->level; ++l) { - size_t len = strlen(lev_config_name(l)); - if (pad < len) - pad = len; - } - - /* extra space */ - pad += 1; - - if (io) { - /* refresh the cached blocks info */ - io_refresh(io); - - printf("\ncached blocks (instant)\n"); - - for (i = state->disklist; i != 0; i = i->next) { - struct snapraid_disk* disk = i->data; - v = disk->cached; - printr(disk->name, pad); - printf("%4" PRIu64 " | ", v); - printc('*', v * USAGE_COLUMN / io->io_max); - printf("\n"); - } - - for (l = 0; l < state->level; ++l) { - v = state->parity[l].cached; - printr(lev_config_name(l), pad); - printf("%4" PRIu64 " | ", v); - printc('*', v * USAGE_COLUMN / io->io_max); - printf("\n"); - } - - printr("", pad); - printf(" +-"); - printc('-', USAGE_COLUMN); - printf("\n"); - } - - printf("\nwait time %% (latest %d seconds)\n", PROGRESS_MAX); - - for (i = state->disklist; i != 0; i = i->next) { - struct snapraid_disk* disk = i->data; - v = disk->progress_tick[current] - disk->progress_tick[oldest]; - printr(disk->name, pad); - printf("%3" PRIu64 "%% | ", v * 100 / tick_total); - printc('*', v * USAGE_COLUMN / tick_total); - printf("\n"); - } - - for (l = 0; l < state->level; ++l) { - v = state->parity[l].progress_tick[current] - state->parity[l].progress_tick[oldest]; - printr(lev_config_name(l), pad); - printf("%3" PRIu64 "%% | ", v * 100 / tick_total); - printc('*', v * USAGE_COLUMN / tick_total); - printf("\n"); - } - - v = state->progress_tick_raid[current] - state->progress_tick_raid[oldest]; - printr("raid", pad); - printf("%3" PRIu64 "%% | ", v * 100 / tick_total); - printc('*', v * USAGE_COLUMN / tick_total); - printf("\n"); - - v = state->progress_tick_hash[current] - state->progress_tick_hash[oldest]; - printr("hash", pad); - printf("%3" PRIu64 "%% | ", v * 100 / tick_total); - printc('*', v * USAGE_COLUMN / tick_total); - printf("\n"); - - v = state->progress_tick_misc[current] - state->progress_tick_misc[oldest]; - printr("misc", pad); - printf("%3" PRIu64 "%% | ", v * 100 / tick_total); - printc('*', v * USAGE_COLUMN / tick_total); - printf("\n"); - - printr("", pad); - printf(" +-"); - printc('-', USAGE_COLUMN); - printf("\n"); -} - void state_usage_print(struct snapraid_state* state) { - uint64_t tick_total; - uint64_t tick_max; - tommy_node* i; - unsigned l; + /* set the latest data */ + state_progress_latest(state); - tick_max = 0; - tick_total = 0; - for (i = state->disklist; i != 0; i = i->next) { - struct snapraid_disk* disk = i->data; - tick_total += disk->tick; - if (disk->tick > tick_max) - tick_max = disk->tick; - } - for (l = 0; l < state->level; ++l) { - tick_total += state->parity[l].tick; - if (state->parity[l].tick > tick_max) - tick_max = state->parity[l].tick; - } - - if (!tick_total) + if (msg_level < MSG_PROGRESS) return; - if (msg_level >= MSG_PROGRESS) { - /* here we don't use msg_progress() because it doesn't allow partial output */ - printf("Time for disk:"); - for (i = state->disklist; i != 0; i = i->next) { - struct snapraid_disk* disk = i->data; - if (disk->tick == tick_max) - printf(" %s:", disk->name); - else - printf(" "); - printf("%" PRIu64 "%%", disk->tick * 100U / tick_total); - } - printf("\n"); - - printf("Time for parity:"); - for (l = 0; l < state->level; ++l) { - if (state->parity[l].tick == tick_max) - printf(" %s:", lev_config_name(l)); - else - printf(" "); - printf("%" PRIu64 "%%", state->parity[l].tick * 100U / tick_total); - } - printf("\n"); - } + /* print a graph for it */ + state_progress_graph(state, 0, state->progress_ptr, PROGRESS_MAX); } void state_fscheck(struct snapraid_state* state, const char* ope) diff --git a/cmdline/state.h b/cmdline/state.h index efcad234..e4fcf72b 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -118,6 +118,7 @@ struct snapraid_option { int force_parity_update; /**< Force parity update even if data is not changed. */ unsigned io_cache; /**< Number of IO buffers to use. 0 for default. */ int auto_conf; /**< Allow to run without configuration file. */ + int force_stats; /**< Force stats print during process. */ }; struct snapraid_state { @@ -359,11 +360,6 @@ void state_usage_parity(struct snapraid_state* state, unsigned* waiting_map, uns */ void state_usage_print(struct snapraid_state* state); -/** - * Print a graph of usage time. - */ -void state_usage_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest); - /** * Check the file-system on all disks. * On error it aborts.