From 5886923bb1dfda820ff20b9fc09bbc7b338694d7 Mon Sep 17 00:00:00 2001 From: Allan Clark Date: Thu, 27 Mar 2025 12:57:41 -0700 Subject: [PATCH] Add bandwidth limit feature --bw-limit --- HISTORY | 4 ++++ Makefile.am | 6 ++++-- cmdline/bw.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ cmdline/bw.h | 45 +++++++++++++++++++++++++++++++++++++++++++++ cmdline/check.c | 12 +++++++++++- cmdline/fnmatch.h | 2 ++ cmdline/handle.c | 3 +++ cmdline/handle.h | 2 ++ cmdline/io.c | 9 +++++++++ cmdline/io.h | 5 +++++ cmdline/mingw.c | 5 +++++ cmdline/mingw.h | 7 +++++++ cmdline/parity.c | 4 ++++ cmdline/parity.h | 2 ++ cmdline/snapraid.c | 36 ++++++++++++++++++++++++++++++++++-- cmdline/state.h | 1 + snapraid.1 | 9 +++++++++ snapraid.d | 6 ++++++ snapraid.txt | 6 ++++++ 19 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 cmdline/bw.c create mode 100644 cmdline/bw.h diff --git a/HISTORY b/HISTORY index f225816f..22fe43e9 100644 --- a/HISTORY +++ b/HISTORY @@ -1,6 +1,10 @@ SnapRAID HISTORY ================ +12.5 2025/XX +============ + * Add a bandwidth limit option --bw-limit [Allan Clark] + 12.4 2025/01 ============ * Avoid a warning about function pointer conversion. No functional changes. diff --git a/Makefile.am b/Makefile.am index 5d835336..9802a4a0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -25,6 +25,7 @@ snapraid_SOURCES = \ tommyds/tommy.c \ cmdline/snapraid.c \ cmdline/io.c \ + cmdline/bw.c \ cmdline/util.c \ cmdline/stream.c \ cmdline/support.c \ @@ -78,6 +79,7 @@ noinst_HEADERS = \ cmdline/portable.h \ cmdline/snapraid.h \ cmdline/io.h \ + cmdline/bw.h \ cmdline/util.h \ cmdline/stream.h \ cmdline/support.h \ @@ -237,7 +239,7 @@ if HAVE_THREAD_CHECKER mv bench/disk3/a/9* bench/disk6/a $(TESTENV) ./mktest$(EXEEXT) change 2 500 bench/disk2/b/* bench/disk3/b/* # Sync again - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync --bw-limit 1M # Other commands that uses threads $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) test-rewrite $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) scrub -p full @@ -288,7 +290,7 @@ endif if HAVE_POSIX $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) pool endif - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check --bw-limit 1M $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -a check #### CONTROLLED #### $(MSG) Filesystem allocation test diff --git a/cmdline/bw.c b/cmdline/bw.c new file mode 100644 index 00000000..e5b59308 --- /dev/null +++ b/cmdline/bw.c @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2016 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "bw.h" + +void bw_init(struct snapraid_bw* bw, uint64_t limit) +{ + bw->limit = limit; + bw->total = 0; + bw->start = tick_ms(); +} + +void bw_limit(struct snapraid_bw* bw, uint64_t bytes) +{ + if (!bw || bw->limit == 0) + return; + + uint64_t elapsed = tick_ms() - bw->start; + uint64_t done; + uint64_t eta; + + done = __atomic_add_fetch(&bw->total, bytes, __ATOMIC_SEQ_CST); + + eta = done * 1000 / bw->limit; + + if (eta > elapsed) { + eta -= elapsed; + usleep(eta * 1000); + } +} diff --git a/cmdline/bw.h b/cmdline/bw.h new file mode 100644 index 00000000..e1456367 --- /dev/null +++ b/cmdline/bw.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2016 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __BW_H +#define __BW_H + +#include "state.h" +#include "support.h" + +/** + * Bandwidth limiting + */ +struct snapraid_bw { + uint64_t limit; /**< Bandwidth limit in bytes per second */ + uint64_t total; /**< Remaining bytes allowed in current second */ + uint64_t start; /**< Time when to reset the bandwidth counter */ +}; + +/** + * Initialize the bandwidth limit + */ +void bw_init(struct snapraid_bw* bw, uint64_t limit); + +/** + * Limit IO bandwidth to stay within the configured limit. + * If no limit is set, returns immediately. + * Otherwise sleeps as needed to maintain the rate limit. + */ +void bw_limit(struct snapraid_bw* bw, uint64_t bytes); + +#endif diff --git a/cmdline/check.c b/cmdline/check.c index 72724e76..05365784 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -888,9 +888,20 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna char esc_buffer[ESC_MAX]; char esc_buffer_alt[ESC_MAX]; bit_vect_t* block_enabled; + struct snapraid_bw bw; handle = handle_mapping(state, &diskmax); + /* initialize the bandwith context */ + bw_init(&bw, state->opt.bwlimit); + + /* share the bandwidth context with all handles */ + for (j = 0; j < diskmax; ++j) + handle[j].bw = &bw; + for (j = 0; j < state->level; ++j) + if (parity[j]) + parity[j]->bw = &bw; + /* we need 1 * data + 2 * parity + 1 * zero */ buffermax = diskmax + 2 * state->level + 1; @@ -2085,4 +2096,3 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b return -1; return 0; } - diff --git a/cmdline/fnmatch.h b/cmdline/fnmatch.h index 0348b53e..6354a480 100644 --- a/cmdline/fnmatch.h +++ b/cmdline/fnmatch.h @@ -47,6 +47,8 @@ extern "C" { #undef FNM_PATHNAME #undef FNM_NOESCAPE #undef FNM_PERIOD +#undef FNM_LEADING_DIR +#undef FNM_CASEFOLD /* Bits set in the FLAGS argument to `fnmatch'. */ #define FNM_PATHNAME (1 << 0) /* No wildcard can ever match `/'. */ diff --git a/cmdline/handle.c b/cmdline/handle.c index e7fa682c..171b20d4 100644 --- a/cmdline/handle.c +++ b/cmdline/handle.c @@ -261,6 +261,8 @@ int handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsigned c count = 0; do { + bw_limit(handle->bw, block_size - count); + /* read the full block to support O_DIRECT */ read_ret = pread(handle->f, block_buffer + count, block_size - count, offset + count); if (read_ret < 0) { @@ -372,6 +374,7 @@ struct snapraid_handle* handle_mapping(struct snapraid_state* state, unsigned* h handle[j].file = 0; handle[j].f = -1; handle[j].valid_size = 0; + handle[j].bw = 0; } /* set the vector */ diff --git a/cmdline/handle.h b/cmdline/handle.h index f9521dbf..570e5f5f 100644 --- a/cmdline/handle.h +++ b/cmdline/handle.h @@ -20,6 +20,7 @@ #include "state.h" #include "support.h" +#include "bw.h" /****************************************************************************/ /* handle */ @@ -33,6 +34,7 @@ struct snapraid_handle { struct advise_struct advise; /**< Advise information. */ data_off_t valid_size; /**< Size of the valid data. */ int created; /**< If the file was created, otherwise it was already existing. */ + struct snapraid_bw* bw; /**< Context for bandwidth limiting. */ }; /** diff --git a/cmdline/io.c b/cmdline/io.c index 3cc77acf..d8fbcd4d 100644 --- a/cmdline/io.c +++ b/cmdline/io.c @@ -877,6 +877,15 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state, io->state = state; + /* initialize bandwidth limiting */ + bw_init(&io->bw, state->opt.bwlimit); + + /* set IO context in handles */ + for (i = 0; i < handle_max; ++i) + handle_map[i].bw = &io->bw; + for (i = 0; i < parity_handle_max; ++i) + parity_handle_map[i].bw = &io->bw; + #if HAVE_THREAD if (io_cache == 0) { /* default is 16 MiB of cache */ diff --git a/cmdline/io.h b/cmdline/io.h index 685ac97c..2a6645e3 100644 --- a/cmdline/io.h +++ b/cmdline/io.h @@ -288,6 +288,11 @@ struct snapraid_io { * Counts the error happening in the writers. */ int writer_error[IO_WRITER_ERROR_MAX]; + + /** + * Bandwidth + */ + struct snapraid_bw bw; }; /** diff --git a/cmdline/mingw.c b/cmdline/mingw.c index 3d23d1ee..b639dcc3 100644 --- a/cmdline/mingw.c +++ b/cmdline/mingw.c @@ -1554,6 +1554,11 @@ void windows_sleep(unsigned seconds) Sleep(seconds * 1000); } +void windows_usleep(uint64_t useconds) +{ + Sleep(useconds / 1000); +} + int windows_link(const char* existing, const char* file) { wchar_t conv_buf_file[CONV_MAX]; diff --git a/cmdline/mingw.h b/cmdline/mingw.h index 8e6b7b1f..0a9ced24 100644 --- a/cmdline/mingw.h +++ b/cmdline/mingw.h @@ -99,6 +99,8 @@ #define stat_desc windows_stat_desc #undef sleep #define sleep windows_sleep +#undef usleep +#define usleep windows_usleep /* 4==DIR, 5,6,7=free, 8==REG */ #define S_IFLNK 0x5000 /* Symbolic link to file */ #define S_ISLNK(m) (((m) & _S_IFMT) == S_IFLNK) @@ -331,6 +333,11 @@ const char* windows_stat_desc(struct stat* st); */ void windows_sleep(unsigned seconds); +/** + * List usleep(). + */ +void windows_usleep(uint64_t useconds); + /** * Like readlink(). */ diff --git a/cmdline/parity.c b/cmdline/parity.c index f75f1532..22b9a4ae 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -864,6 +864,8 @@ int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigne if (split->valid_size < offset + block_size) split->valid_size = offset + block_size; + bw_limit(handle->bw, block_size); + write_ret = pwrite(split->f, block_buffer, block_size, offset); if (write_ret != (ssize_t)block_size) { /* conversion is safe because block_size is always small */ /* LCOV_EXCL_START */ @@ -915,6 +917,8 @@ int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned count = 0; do { + bw_limit(handle->bw, block_size - count); + read_ret = pread(split->f, block_buffer + count, block_size - count, offset + count); if (read_ret < 0) { /* LCOV_EXCL_START */ diff --git a/cmdline/parity.h b/cmdline/parity.h index 3380eb7d..41bf6af8 100644 --- a/cmdline/parity.h +++ b/cmdline/parity.h @@ -19,6 +19,7 @@ #define __PARITY_H #include "support.h" +#include "bw.h" /****************************************************************************/ /* parity */ @@ -57,6 +58,7 @@ struct snapraid_parity_handle { struct snapraid_split_handle split_map[SPLIT_MAX]; unsigned split_mac; /**< Number of parity splits. */ unsigned level; /**< Level of the parity. */ + struct snapraid_bw* bw; /**< Context for bandwidth limiting. */ }; /** diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index e1ac8524..4b26853f 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -74,6 +74,7 @@ void usage(void) printf(" " SWITCH_GETOPT_LONG("-N, --force-nocopy ", "-N") " Force commands disabling the copy detection\n"); printf(" " SWITCH_GETOPT_LONG("-F, --force-full ", "-F") " Force a full parity computation in sync\n"); printf(" " SWITCH_GETOPT_LONG("-R, --force-realloc ", "-R") " Force a full parity reallocation in sync\n"); + printf(" " SWITCH_GETOPT_LONG("-w, --bw-limit RATE ", "-w") " Limit IO bandwidth (M|G)\n"); printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n"); } @@ -333,6 +334,7 @@ struct option long_options[] = { { "force-nocopy", 0, 0, 'N' }, { "force-full", 0, 0, 'F' }, { "force-realloc", 0, 0, 'R' }, + { "bw-limit", 1, 0, 'w' }, { "audit-only", 0, 0, 'a' }, { "pre-hash", 0, 0, 'h' }, { "speed-test", 0, 0, 'T' }, /* undocumented speed test command */ @@ -484,7 +486,7 @@ struct option long_options[] = { }; #endif -#define OPTIONS "c:f:d:mebp:o:S:B:L:i:l:ZEUDNFRahTC:vqHVG" +#define OPTIONS "c:f:d:mebp:o:S:B:L:i:l:ZEUDNFRahTC:vqHVGw:" volatile int global_interrupt = 0; @@ -673,6 +675,37 @@ int main(int argc, char* argv[]) /* LCOV_EXCL_STOP */ } break; + case 'w' : /* --bw-limit */ + if (optarg == 0) { + /* LCOV_EXCL_START */ + log_fatal("Missing bandwidth limit\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* Parse the number part */ + opt.bwlimit = strtoul(optarg, &e, 10); + if (!e || e == optarg) { + /* LCOV_EXCL_START */ + log_fatal("Invalid bandwidth limit '%s'\n", optarg); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* Handle suffixes */ + if ((e[0] == 'k' || e[0] == 'K') && e[1] == 0) { + opt.bwlimit *= 1000; + } else if ((e[0] == 'm' || e[0] == 'M') && e[1] == 0) { + opt.bwlimit *= 1000 * 1000; + } else if ((e[0] == 'g' || e[0] == 'G') && e[1] == 0) { + opt.bwlimit *= 1000 * 1000 * 1000; + } else if (e[0] != '\0') { + /* LCOV_EXCL_START */ + log_fatal("Invalid bandwidth limit suffix '%s'\n", e); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + break; case 'S' : blockstart = strtoul(optarg, &e, 0); if (!e || *e) { @@ -1543,4 +1576,3 @@ int main(int argc, char* argv[]) return EXIT_SUCCESS; } - diff --git a/cmdline/state.h b/cmdline/state.h index d9e30bdd..77af8620 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -120,6 +120,7 @@ struct snapraid_option { int force_stats; /**< Force stats print during process. */ uint64_t parity_limit_size; /**< Test limit for parity files. */ int skip_multi_scan; /**< Don't use threads in scan. */ + uint64_t bwlimit; /**< Bandwidth limit in bytes per second. */ }; struct snapraid_state { diff --git a/snapraid.1 b/snapraid.1 index c18028e3..3543449c 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -25,6 +25,10 @@ snapraid [\-c, \-\-conf CONFIG] [\-o, \-\-older\-than DAYS] [\-l, \-\-log FILE] .PD 0 .PP +.PD + [\-w, \-\-bw\-limit RATE] +.PD 0 +.PP .PD [\-Z, \-\-force\-zero] [\-E, \-\-force\-empty] .PD 0 @@ -1092,6 +1096,11 @@ The files are read also in sub\-directories and they are identified regardless of their name. This option can be used only with \[dq]check\[dq] and \[dq]fix\[dq]. .TP +.B \-w, \-\-bw\-limit RATE +Applies a global bandwidth limit for all disks. The RATE is +the number of bytes per second. You can specify a multiplier +as K, M, or G (e.g., \-\-bw\-limit 1G). +.TP .B \-Z, \-\-force\-zero Forces the insecure operation of syncing a file with zero size that before was not. diff --git a/snapraid.d b/snapraid.d index b41f5b50..cfe49df7 100644 --- a/snapraid.d +++ b/snapraid.d @@ -8,6 +8,7 @@ Synopsis : [-a, --audit-only] [-h, --pre-hash] [-i, --import DIR] : [-p, --plan PERC|bad|new|full] : [-o, --older-than DAYS] [-l, --log FILE] + : [-w, --bw-limit RATE] : [-Z, --force-zero] [-E, --force-empty] : [-U, --force-uuid] [-D, --force-device] : [-N, --force-nocopy] [-F, --force-full] @@ -788,6 +789,11 @@ Options identified regardless of their name. This option can be used only with "check" and "fix". + -w, --bw-limit RATE + Applies a global bandwidth limit for all disks. The RATE is + the number of bytes per second. You can specify a multiplier + as K, M, or G (e.g., --bw-limit 1G). + -Z, --force-zero Forces the insecure operation of syncing a file with zero size that before was not. diff --git a/snapraid.txt b/snapraid.txt index 044f22bf..32d62c4f 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -12,6 +12,7 @@ snapraid [-c, --conf CONFIG] [-a, --audit-only] [-h, --pre-hash] [-i, --import DIR] [-p, --plan PERC|bad|new|full] [-o, --older-than DAYS] [-l, --log FILE] + [-w, --bw-limit RATE] [-Z, --force-zero] [-E, --force-empty] [-U, --force-uuid] [-D, --force-device] [-N, --force-nocopy] [-F, --force-full] @@ -845,6 +846,11 @@ SnapRAID provides the following options: identified regardless of their name. This option can be used only with "check" and "fix". + -w, --bw-limit RATE + Applies a global bandwidth limit for all disks. The RATE is + the number of bytes per second. You can specify a multiplier + as K, M, or G (e.g., --bw-limit 1G). + -Z, --force-zero Forces the insecure operation of syncing a file with zero size that before was not.