rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add new locate command with --tail option.

Browse source 
Co-authored-by: Andrea Mazzoleni <amadvance@gmail.com>
This commit is contained in:
Ralf 2026-02-02 11:19:51 +01:00 • committed by Andrea Mazzoleni
parent 1c4f9870fb
commit c09bdc572e
8 changed files with 291 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

5
HISTORY
View file

@ -13,6 +13,11 @@ SnapRAID HISTORY
parity, mirroring the workflow of .gitignore.
* Added support for the ** globbing character to allow recursive pattern
matching across multiple directory levels.
* Added a new locate command to map physical file offsets within the
parity volume. This feature facilitates diagnostic analysis of the parity
distribution. It supports the -t, --tail option to filter files located
specifically at the end of the parity file. This option handles size
suffixes K, M, and G using decimal SI units [Ralf1108].
* Added detection of the smartctl executable. Improves reliability when running
under sudo or in environments where /sbin and /usr/sbin are not in PATH.
* Avoided saving the content file when it was not strictly necessary.

4
Makefile.am
View file

@ -43,6 +43,7 @@ snapraid_SOURCES = \
cmdline/list.c \
cmdline/pool.c \
cmdline/parity.c \
cmdline/locate.c \
cmdline/handle.c \
cmdline/touch.c \
cmdline/device.c \
@ -87,6 +88,7 @@ noinst_HEADERS = \
cmdline/elem.h \
cmdline/state.h \
cmdline/parity.h \
cmdline/locate.h \
cmdline/handle.h \
cmdline/murmur3.c \
cmdline/murmur3test.c \
@ -556,6 +558,8 @@ endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) list --test-fmt file > output.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) list --test-fmt disk > output.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) list --test-fmt path > output.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) locate --test-fmt file > output.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) locate -t 10K --test-fmt path > output.log
if HAVE_POSIX
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) pool
endif

17
cmdline/elem.c
View file

@ -19,6 +19,7 @@
#include "elem.h"
#include "support.h"
#include "state.h"
#include "util.h"
/****************************************************************************/
@ -1461,3 +1462,19 @@ const char* fmt_term(const struct snapraid_disk* disk, const char* str, char* bu
}
}
const char* fmt_size(uint64_t size, char* buffer, size_t buffer_size)
{
if (size >= 10UL * TERA)
snprintf(buffer, buffer_size, "%" PRIu64 " T", size / TERA);
else if (size >= 10UL * GIGA)
snprintf(buffer, buffer_size, "%" PRIu64 " G", size / GIGA);
else if (size >= 10UL * MEGA)
snprintf(buffer, buffer_size, "%" PRIu64 " M", size / MEGA);
else if (size >= 10UL * KILO)
snprintf(buffer, buffer_size, "%" PRIu64 " k", size / KILO);
else
snprintf(buffer, buffer_size, "%" PRIu64 " ", size);
return buffer;
}

5
cmdline/elem.h
View file

@ -1257,5 +1257,10 @@ const char* fmt_poll(const struct snapraid_disk* disk, const char* str, char* bu
*/
const char* fmt_term(const struct snapraid_disk* disk, const char* str, char* buffer);
/**
* Format a size adding suffix K/M/G/T
*/
const char* fmt_size(uint64_t size, char* buffer, size_t buffer_size);
#endif

181
cmdline/locate.c Normal file
View file

@ -0,0 +1,181 @@
/*
* Copyright (C) 2026 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "portable.h"
#include "elem.h"
#include "parity.h"
#include "state.h"
#include "stream.h"
#include "locate.h"
struct snapraid_parity_entry {
block_off_t block_size;
block_off_t low; /**< Lower position in the parity */
block_off_t high; /**< Higher position in the parity */
block_off_t fragments; /**< Number of fragments in the parity */
struct snapraid_file* file;
struct snapraid_disk* disk;
tommy_node node;
};
static int parity_entry_compare(const void* void_a, const void* void_b)
{
const struct snapraid_parity_entry* entry_a = void_a;
const struct snapraid_parity_entry* entry_b = void_b;
if (entry_a->high < entry_b->high)
return -1;
if (entry_a->high > entry_b->high)
return 1;
if (entry_a->low < entry_b->low)
return -1;
if (entry_a->low > entry_b->low)
return 1;
return strcmp(entry_a->file->sub, entry_b->file->sub);
}
static void dump_entry(void* void_entry)
{
char esc_buffer[ESC_MAX];
struct snapraid_parity_entry* entry = void_entry;
printf("%12" PRIu64 " ", entry->low * (uint64_t)entry->block_size);
printf("%12" PRIu64 " ", (entry->high - entry->low + 1) * (uint64_t)entry->block_size);
printf("%8u ", entry->fragments);
printf("%s\n", fmt_term(entry->disk, entry->file->sub, esc_buffer));
}
static void add_size_to_sum(void* arg, void* void_entry)
{
struct snapraid_parity_entry* entry = void_entry;
data_off_t* size_sum = arg;
*size_sum += entry->file->size;
}
static void collect_parity_block_file(uint32_t block_size, struct snapraid_disk* disk, struct snapraid_file* file, tommy_list* file_list, block_off_t low_occupied_block_number)
{
block_off_t parity_low = 0; /* lower block */
block_off_t parity_high = 0; /* higher block */
block_off_t parity_fragments = 0; /* number of fragments */
for (block_off_t i = 0; i < file->blockmax; ++i) {
block_off_t parity_pos = fs_file2par_find(disk, file, i);
/* check if a valid position is found */
if (parity_pos == POS_NULL) {
/* block not yet allocated */
continue;
}
if (parity_fragments == 0) {
parity_low = parity_pos;
parity_high = parity_pos;
parity_fragments = 1;
} else {
if (parity_pos != parity_high + 1)
++parity_fragments;
parity_high = parity_pos;
}
}
if (parity_fragments == 0)
return; /* not allocated */
if (low_occupied_block_number > 0 && parity_high < low_occupied_block_number)
return; /* entry not relevant */
/* found a relevant block so add the corresponding file */
struct snapraid_parity_entry* entry = malloc_nofail(sizeof(struct snapraid_parity_entry));
entry->block_size = block_size;
entry->file = file;
entry->disk = disk;
entry->low = parity_low;
entry->high = parity_high;
entry->fragments = parity_fragments;
tommy_list_insert_tail(file_list, &entry->node, entry);
}
void state_locate(struct snapraid_state* state, uint64_t parity_tail)
{
char buf[64];
uint64_t block_size = state->block_size;
block_off_t min_occupied_block_number;
printf("SnapRAID locate report:\n");
printf("\n");
if (parity_tail == 0) {
printf("Locate all files\n\n");
min_occupied_block_number = 0;
} else {
printf("Locate files within the tail of %sB of the parity\n\n", fmt_size(parity_tail, buf, sizeof(buf)));
data_off_t block_max = parity_allocated_size(state);
data_off_t parity_size = block_max * block_size;
printf("Current parity size is %sB\n", fmt_size(parity_size, buf, sizeof(buf)));
block_off_t tail_block = (parity_tail + block_size - 1) / block_size;
if (tail_block >= block_max) {
printf("Specified tail greater than the parity size!\n");
return;
}
min_occupied_block_number = block_max - tail_block;
}
msg_progress("Collecting files with offset greater or equal to %" PRIu64 "\n", min_occupied_block_number * block_size);
tommy_list files;
tommy_list_init(&files);
for (tommy_node* i = tommy_list_head(&state->disklist); i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
for (tommy_node* j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
struct snapraid_file* file = j->data;
collect_parity_block_file(block_size, disk, file, &files, min_occupied_block_number);
}
}
printf("\n");
if (tommy_list_count(&files) == 0) {
printf("No files located in the specified parity tail.\n");
} else {
tommy_list_sort(&files, parity_entry_compare);
data_off_t total_size_located = 0;
tommy_list_foreach_arg(&files, add_size_to_sum, &total_size_located);
printf("Located data in this range: %sB\n", fmt_size(total_size_located, buf, sizeof(buf)));
printf("\n");
/* |<##################################################################72>|####80>| */
printf(" Offset Span Frags\n");
tommy_list_foreach(&files, dump_entry);
}
tommy_list_foreach(&files, free);
}

29
cmdline/locate.h Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright (C) 2026 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __LOCATE_H
#define __LOCATE_H
/****************************************************************************/
/* snapraid */
#include "state.h"
void state_locate(struct snapraid_state* state, uint64_t parity_tail);
#endif

67
cmdline/snapraid.c
View file

@ -25,6 +25,7 @@
#include "state.h"
#include "io.h"
#include "raid/raid.h"
#include "locate.h"
/****************************************************************************/
/* misc */
@ -714,6 +715,7 @@ static struct option long_options[] = {
{ "bw-limit", 1, 0, 'w' },
{ "audit-only", 0, 0, 'a' },
{ "pre-hash", 0, 0, 'h' },
{ "tail", 1, 0, 't' },
{ "speed-test", 0, 0, 'T' }, /* undocumented speed test command */
{ "speed-test-period", 1, 0, OPT_TEST_SPEED_PERIOD }, /* for how many milliseconds test each feature. Default 1000. */
{ "speed-test-disks-number", 1, 0, OPT_TEST_SPEED_DISKS_NUMBER }, /* how many disk number uses in the speed test. Default 8. */
@ -868,12 +870,38 @@ static struct option long_options[] = {
#endif
/*
* Free letters: gIjJkKMnPQrtuxXWz
* Free letters: gIjJkKMnPQruxXWz
*
* The 's' letter is used in main.c
* The 'G' letter is free but only from 14.0
*/
#define OPTIONS "c:f:d:mebp:o:S:B:L:i:l:AZEUDNFRahTC:vqHVw:"
#define OPTIONS "t:c:f:d:mebp:o:S:B:L:i:l:AZEUDNFRahTC:vqHVw:"
int parse_option_size(const char* arg, uint64_t* out_size)
{
char* e;
/* parse the number part */
data_off_t size = strtoul(optarg, &e, 10);
if (!e || e == arg)
return -1;
/* Handle suffixes */
if ((e[0] == 'k' || e[0] == 'K') && e[1] == 0) {
size *= KILO;
} else if ((e[0] == 'm' || e[0] == 'M') && e[1] == 0) {
size *= MEGA;
} else if ((e[0] == 'g' || e[0] == 'G') && e[1] == 0) {
size *= GIGA;
} else if ((e[0] == 't' || e[0] == 'T') && e[1] == 0) {
size *= TERA;
} else if (e[0] != '\0') {
return -1;
}
*out_size = size;
return 0;
}
volatile int global_interrupt = 0;
@ -926,6 +954,7 @@ void signal_init(void)
#define OPERATION_DEVICES 16
#define OPERATION_SMART 17
#define OPERATION_PROBE 18
#define OPERATION_LOCATE 19
int snapraid_main(int argc, char* argv[])
{
@ -957,6 +986,7 @@ int snapraid_main(int argc, char* argv[])
int speed_test_blocks_size;
time_t t;
struct tm* tm;
uint64_t parity_tail = 0;
#if HAVE_LOCALTIME_R
struct tm tm_res;
#endif
@ -1074,28 +1104,12 @@ int snapraid_main(int argc, char* argv[])
/* LCOV_EXCL_STOP */
}
/* Parse the number part */
opt.bwlimit = strtoul(optarg, &e, 10);
if (!e || e == optarg) {
if (parse_option_size(optarg, &opt.bwlimit) != 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid bandwidth limit '%s'\n", optarg);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* Handle suffixes */
if ((e[0] == 'k' || e[0] == 'K') && e[1] == 0) {
opt.bwlimit *= 1000;
} else if ((e[0] == 'm' || e[0] == 'M') && e[1] == 0) {
opt.bwlimit *= 1000 * 1000;
} else if ((e[0] == 'g' || e[0] == 'G') && e[1] == 0) {
opt.bwlimit *= 1000 * 1000 * 1000;
} else if (e[0] != '\0') {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid bandwidth limit suffix '%s'\n", e);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
break;
case 'S' :
blockstart = strtoul(optarg, &e, 0);
@ -1133,6 +1147,14 @@ int snapraid_main(int argc, char* argv[])
}
import_timestamp = optarg;
break;
case 't' :
if (parse_option_size(optarg, &parity_tail) != 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid tail size '%s'\n", optarg);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
break;
case OPT_TEST_IMPORT_CONTENT :
if (import_content) {
/* LCOV_EXCL_START */
@ -1448,6 +1470,8 @@ int snapraid_main(int argc, char* argv[])
operation = OPERATION_SMART;
} else if (strcmp(argv[optind], "probe") == 0) {
operation = OPERATION_PROBE;
} else if (strcmp(argv[optind], "locate") == 0) {
operation = OPERATION_LOCATE;
} else {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Unknown command '%s'\n", argv[optind]);
@ -1900,6 +1924,11 @@ int snapraid_main(int argc, char* argv[])
memory();
state_status(&state);
} else if (operation == OPERATION_LOCATE) {
state_read(&state);
state_locate(&state, parity_tail);
} else if (operation == OPERATION_DUP) {
state_read(&state);

4
cmdline/state.h
View file

@ -50,11 +50,11 @@ const char* lev_config_name(unsigned level);
#define KILO (1000)
#define MEGA (1000 * 1000)
#define GIGA (1000 * 1000 * 1000)
#define TERA (1000 * 1000 * 1000 * 1000LL)
#define TERA (1000 * 1000 * 1000 * 1000UL)
#define KIBI (1024)
#define MEBI (1024 * 1024)
#define GIBI (1024 * 1024 * 1024)
#define TEBI (1024 * 1024 * 1024 * 1024LL)
#define TEBI (1024 * 1024 * 1024 * 1024UL)
#define SORT_PHYSICAL 1 /**< Sort by physical order. */
#define SORT_INODE 2 /**< Sort by inode. */