diff --git a/HISTORY b/HISTORY index 3a1e889f..bc222f58 100644 --- a/HISTORY +++ b/HISTORY @@ -13,6 +13,11 @@ SnapRAID HISTORY parity, mirroring the workflow of .gitignore. * Added support for the ** globbing character to allow recursive pattern matching across multiple directory levels. + * Added a new locate command to map physical file offsets within the + parity volume. This feature facilitates diagnostic analysis of the parity + distribution. It supports the -t, --tail option to filter files located + specifically at the end of the parity file. This option handles size + suffixes K, M, and G using decimal SI units [Ralf1108]. * Added detection of the smartctl executable. Improves reliability when running under sudo or in environments where /sbin and /usr/sbin are not in PATH. * Avoided saving the content file when it was not strictly necessary. diff --git a/Makefile.am b/Makefile.am index 4f3efd4e..689f6bd4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -43,6 +43,7 @@ snapraid_SOURCES = \ cmdline/list.c \ cmdline/pool.c \ cmdline/parity.c \ + cmdline/locate.c \ cmdline/handle.c \ cmdline/touch.c \ cmdline/device.c \ @@ -87,6 +88,7 @@ noinst_HEADERS = \ cmdline/elem.h \ cmdline/state.h \ cmdline/parity.h \ + cmdline/locate.h \ cmdline/handle.h \ cmdline/murmur3.c \ cmdline/murmur3test.c \ @@ -556,6 +558,8 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) list --test-fmt file > output.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) list --test-fmt disk > output.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) list --test-fmt path > output.log + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) locate --test-fmt file > output.log + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) locate -t 10K --test-fmt path > output.log if HAVE_POSIX $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) pool endif diff --git a/cmdline/elem.c b/cmdline/elem.c index 2abaa5f3..a046878e 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -19,6 +19,7 @@ #include "elem.h" #include "support.h" +#include "state.h" #include "util.h" /****************************************************************************/ @@ -1461,3 +1462,19 @@ const char* fmt_term(const struct snapraid_disk* disk, const char* str, char* bu } } +const char* fmt_size(uint64_t size, char* buffer, size_t buffer_size) +{ + if (size >= 10UL * TERA) + snprintf(buffer, buffer_size, "%" PRIu64 " T", size / TERA); + else if (size >= 10UL * GIGA) + snprintf(buffer, buffer_size, "%" PRIu64 " G", size / GIGA); + else if (size >= 10UL * MEGA) + snprintf(buffer, buffer_size, "%" PRIu64 " M", size / MEGA); + else if (size >= 10UL * KILO) + snprintf(buffer, buffer_size, "%" PRIu64 " k", size / KILO); + else + snprintf(buffer, buffer_size, "%" PRIu64 " ", size); + + return buffer; +} + diff --git a/cmdline/elem.h b/cmdline/elem.h index f8381aa5..c33c9fc2 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -1257,5 +1257,10 @@ const char* fmt_poll(const struct snapraid_disk* disk, const char* str, char* bu */ const char* fmt_term(const struct snapraid_disk* disk, const char* str, char* buffer); +/** + * Format a size adding suffix K/M/G/T + */ +const char* fmt_size(uint64_t size, char* buffer, size_t buffer_size); + #endif diff --git a/cmdline/locate.c b/cmdline/locate.c new file mode 100644 index 00000000..29aece03 --- /dev/null +++ b/cmdline/locate.c @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2026 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "elem.h" +#include "parity.h" +#include "state.h" +#include "stream.h" +#include "locate.h" + +struct snapraid_parity_entry { + block_off_t block_size; + block_off_t low; /**< Lower position in the parity */ + block_off_t high; /**< Higher position in the parity */ + block_off_t fragments; /**< Number of fragments in the parity */ + struct snapraid_file* file; + struct snapraid_disk* disk; + tommy_node node; +}; + +static int parity_entry_compare(const void* void_a, const void* void_b) +{ + const struct snapraid_parity_entry* entry_a = void_a; + const struct snapraid_parity_entry* entry_b = void_b; + + if (entry_a->high < entry_b->high) + return -1; + if (entry_a->high > entry_b->high) + return 1; + + if (entry_a->low < entry_b->low) + return -1; + if (entry_a->low > entry_b->low) + return 1; + + return strcmp(entry_a->file->sub, entry_b->file->sub); +} + +static void dump_entry(void* void_entry) +{ + char esc_buffer[ESC_MAX]; + struct snapraid_parity_entry* entry = void_entry; + printf("%12" PRIu64 " ", entry->low * (uint64_t)entry->block_size); + printf("%12" PRIu64 " ", (entry->high - entry->low + 1) * (uint64_t)entry->block_size); + printf("%8u ", entry->fragments); + printf("%s\n", fmt_term(entry->disk, entry->file->sub, esc_buffer)); +} + +static void add_size_to_sum(void* arg, void* void_entry) +{ + struct snapraid_parity_entry* entry = void_entry; + data_off_t* size_sum = arg; + + *size_sum += entry->file->size; +} + +static void collect_parity_block_file(uint32_t block_size, struct snapraid_disk* disk, struct snapraid_file* file, tommy_list* file_list, block_off_t low_occupied_block_number) +{ + block_off_t parity_low = 0; /* lower block */ + block_off_t parity_high = 0; /* higher block */ + block_off_t parity_fragments = 0; /* number of fragments */ + + for (block_off_t i = 0; i < file->blockmax; ++i) { + block_off_t parity_pos = fs_file2par_find(disk, file, i); + + /* check if a valid position is found */ + if (parity_pos == POS_NULL) { + /* block not yet allocated */ + continue; + } + + if (parity_fragments == 0) { + parity_low = parity_pos; + parity_high = parity_pos; + parity_fragments = 1; + } else { + if (parity_pos != parity_high + 1) + ++parity_fragments; + parity_high = parity_pos; + } + } + + if (parity_fragments == 0) + return; /* not allocated */ + + if (low_occupied_block_number > 0 && parity_high < low_occupied_block_number) + return; /* entry not relevant */ + + /* found a relevant block so add the corresponding file */ + struct snapraid_parity_entry* entry = malloc_nofail(sizeof(struct snapraid_parity_entry)); + entry->block_size = block_size; + entry->file = file; + entry->disk = disk; + entry->low = parity_low; + entry->high = parity_high; + entry->fragments = parity_fragments; + tommy_list_insert_tail(file_list, &entry->node, entry); +} + +void state_locate(struct snapraid_state* state, uint64_t parity_tail) +{ + char buf[64]; + uint64_t block_size = state->block_size; + + block_off_t min_occupied_block_number; + + printf("SnapRAID locate report:\n"); + printf("\n"); + + if (parity_tail == 0) { + printf("Locate all files\n\n"); + min_occupied_block_number = 0; + } else { + printf("Locate files within the tail of %sB of the parity\n\n", fmt_size(parity_tail, buf, sizeof(buf))); + + data_off_t block_max = parity_allocated_size(state); + data_off_t parity_size = block_max * block_size; + + printf("Current parity size is %sB\n", fmt_size(parity_size, buf, sizeof(buf))); + + block_off_t tail_block = (parity_tail + block_size - 1) / block_size; + + if (tail_block >= block_max) { + printf("Specified tail greater than the parity size!\n"); + return; + } + + min_occupied_block_number = block_max - tail_block; + } + + msg_progress("Collecting files with offset greater or equal to %" PRIu64 "\n", min_occupied_block_number * block_size); + + tommy_list files; + tommy_list_init(&files); + for (tommy_node* i = tommy_list_head(&state->disklist); i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + for (tommy_node* j = tommy_list_head(&disk->filelist); j != 0; j = j->next) { + struct snapraid_file* file = j->data; + collect_parity_block_file(block_size, disk, file, &files, min_occupied_block_number); + } + } + + printf("\n"); + + if (tommy_list_count(&files) == 0) { + printf("No files located in the specified parity tail.\n"); + } else { + tommy_list_sort(&files, parity_entry_compare); + + data_off_t total_size_located = 0; + + tommy_list_foreach_arg(&files, add_size_to_sum, &total_size_located); + + printf("Located data in this range: %sB\n", fmt_size(total_size_located, buf, sizeof(buf))); + + printf("\n"); + + /* |<##################################################################72>|####80>| */ + printf(" Offset Span Frags\n"); + + tommy_list_foreach(&files, dump_entry); + } + + tommy_list_foreach(&files, free); +} + diff --git a/cmdline/locate.h b/cmdline/locate.h new file mode 100644 index 00000000..b9c99ddf --- /dev/null +++ b/cmdline/locate.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2026 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __LOCATE_H +#define __LOCATE_H + +/****************************************************************************/ +/* snapraid */ + +#include "state.h" + +void state_locate(struct snapraid_state* state, uint64_t parity_tail); + +#endif + diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 5c5b1330..18400940 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -25,6 +25,7 @@ #include "state.h" #include "io.h" #include "raid/raid.h" +#include "locate.h" /****************************************************************************/ /* misc */ @@ -714,6 +715,7 @@ static struct option long_options[] = { { "bw-limit", 1, 0, 'w' }, { "audit-only", 0, 0, 'a' }, { "pre-hash", 0, 0, 'h' }, + { "tail", 1, 0, 't' }, { "speed-test", 0, 0, 'T' }, /* undocumented speed test command */ { "speed-test-period", 1, 0, OPT_TEST_SPEED_PERIOD }, /* for how many milliseconds test each feature. Default 1000. */ { "speed-test-disks-number", 1, 0, OPT_TEST_SPEED_DISKS_NUMBER }, /* how many disk number uses in the speed test. Default 8. */ @@ -868,12 +870,38 @@ static struct option long_options[] = { #endif /* - * Free letters: gIjJkKMnPQrtuxXWz + * Free letters: gIjJkKMnPQruxXWz * * The 's' letter is used in main.c * The 'G' letter is free but only from 14.0 */ -#define OPTIONS "c:f:d:mebp:o:S:B:L:i:l:AZEUDNFRahTC:vqHVw:" +#define OPTIONS "t:c:f:d:mebp:o:S:B:L:i:l:AZEUDNFRahTC:vqHVw:" + +int parse_option_size(const char* arg, uint64_t* out_size) +{ + char* e; + + /* parse the number part */ + data_off_t size = strtoul(optarg, &e, 10); + if (!e || e == arg) + return -1; + + /* Handle suffixes */ + if ((e[0] == 'k' || e[0] == 'K') && e[1] == 0) { + size *= KILO; + } else if ((e[0] == 'm' || e[0] == 'M') && e[1] == 0) { + size *= MEGA; + } else if ((e[0] == 'g' || e[0] == 'G') && e[1] == 0) { + size *= GIGA; + } else if ((e[0] == 't' || e[0] == 'T') && e[1] == 0) { + size *= TERA; + } else if (e[0] != '\0') { + return -1; + } + + *out_size = size; + return 0; +} volatile int global_interrupt = 0; @@ -926,6 +954,7 @@ void signal_init(void) #define OPERATION_DEVICES 16 #define OPERATION_SMART 17 #define OPERATION_PROBE 18 +#define OPERATION_LOCATE 19 int snapraid_main(int argc, char* argv[]) { @@ -957,6 +986,7 @@ int snapraid_main(int argc, char* argv[]) int speed_test_blocks_size; time_t t; struct tm* tm; + uint64_t parity_tail = 0; #if HAVE_LOCALTIME_R struct tm tm_res; #endif @@ -1074,28 +1104,12 @@ int snapraid_main(int argc, char* argv[]) /* LCOV_EXCL_STOP */ } - /* Parse the number part */ - opt.bwlimit = strtoul(optarg, &e, 10); - if (!e || e == optarg) { + if (parse_option_size(optarg, &opt.bwlimit) != 0) { /* LCOV_EXCL_START */ log_fatal(EUSER, "Invalid bandwidth limit '%s'\n", optarg); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } - - /* Handle suffixes */ - if ((e[0] == 'k' || e[0] == 'K') && e[1] == 0) { - opt.bwlimit *= 1000; - } else if ((e[0] == 'm' || e[0] == 'M') && e[1] == 0) { - opt.bwlimit *= 1000 * 1000; - } else if ((e[0] == 'g' || e[0] == 'G') && e[1] == 0) { - opt.bwlimit *= 1000 * 1000 * 1000; - } else if (e[0] != '\0') { - /* LCOV_EXCL_START */ - log_fatal(EUSER, "Invalid bandwidth limit suffix '%s'\n", e); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } break; case 'S' : blockstart = strtoul(optarg, &e, 0); @@ -1133,6 +1147,14 @@ int snapraid_main(int argc, char* argv[]) } import_timestamp = optarg; break; + case 't' : + if (parse_option_size(optarg, &parity_tail) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EUSER, "Invalid tail size '%s'\n", optarg); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + break; case OPT_TEST_IMPORT_CONTENT : if (import_content) { /* LCOV_EXCL_START */ @@ -1448,6 +1470,8 @@ int snapraid_main(int argc, char* argv[]) operation = OPERATION_SMART; } else if (strcmp(argv[optind], "probe") == 0) { operation = OPERATION_PROBE; + } else if (strcmp(argv[optind], "locate") == 0) { + operation = OPERATION_LOCATE; } else { /* LCOV_EXCL_START */ log_fatal(EUSER, "Unknown command '%s'\n", argv[optind]); @@ -1900,6 +1924,11 @@ int snapraid_main(int argc, char* argv[]) memory(); state_status(&state); + } else if (operation == OPERATION_LOCATE) { + state_read(&state); + + state_locate(&state, parity_tail); + } else if (operation == OPERATION_DUP) { state_read(&state); diff --git a/cmdline/state.h b/cmdline/state.h index 5937f7b9..534dce76 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -50,11 +50,11 @@ const char* lev_config_name(unsigned level); #define KILO (1000) #define MEGA (1000 * 1000) #define GIGA (1000 * 1000 * 1000) -#define TERA (1000 * 1000 * 1000 * 1000LL) +#define TERA (1000 * 1000 * 1000 * 1000UL) #define KIBI (1024) #define MEBI (1024 * 1024) #define GIBI (1024 * 1024 * 1024) -#define TEBI (1024 * 1024 * 1024 * 1024LL) +#define TEBI (1024 * 1024 * 1024 * 1024UL) #define SORT_PHYSICAL 1 /**< Sort by physical order. */ #define SORT_INODE 2 /**< Sort by inode. */