rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

More efficient info bucket management

Browse source 
Logged at every state_read() and state_write()
This commit is contained in:
Andrea Mazzoleni 2026-02-08 22:56:22 +01:00
parent 83b1c9bf3b
commit d5b5a9bcbf
13 changed files with 375 additions and 495 deletions
Download patch file Download diff file Expand all files Collapse all files

69
cmdline/elem.c
View file

@ -1413,18 +1413,75 @@ void map_free(struct snapraid_map* map)
free(map);
}
int time_compare(const void* void_a, const void* void_b)
{
const time_t* time_a = void_a;
const time_t* time_b = void_b;
/****************************************************************************/
/* bucket */
if (*time_a < *time_b)
static int bucket_cmp(const void* void_a, const void* void_b)
{
const struct snapraid_bucket* a = void_a;
const struct snapraid_bucket* b = void_b;
if (a->time_at < b->time_at)
return -1;
if (*time_a > *time_b)
if (a->time_at > b->time_at)
return 1;
return 0;
}
void bucket_free(struct snapraid_bucket* bucket)
{
free(bucket);
}
void bucket_insert(tommy_hashdyn* bucket_hash, time_t time_at, block_off_t count, int justsynced)
{
unsigned hash = tommy_inthash_u32(time_at);
tommy_node* i = tommy_hashdyn_bucket(bucket_hash, hash);
while (i) {
struct snapraid_bucket* entry = i->data;
if (entry->time_at == time_at) {
if (justsynced)
entry->count_justsynced += count;
else
entry->count_scrubbed += count;
return;
}
i = i->next;
}
struct snapraid_bucket* entry = malloc_nofail(sizeof(struct snapraid_bucket));
entry->time_at = time_at;
if (justsynced) {
entry->count_justsynced = count;
entry->count_scrubbed = 0;
} else {
entry->count_justsynced = 0;
entry->count_scrubbed = count;
}
tommy_hashdyn_insert(bucket_hash, &entry->node, entry, hash);
}
void bucket_to_list(tommy_hashdyn* bucket_hash, tommy_list* bucket_list, block_off_t* bucketcount)
{
/* clear previous list */
tommy_list_foreach(bucket_list, (tommy_foreach_func*)bucket_free);
tommy_list_init(bucket_list);
log_tag("content_info:bucket_count:%" PRIu64 "\n", (uint64_t)tommy_hashdyn_count(bucket_hash));
tommy_hashdyn_to_list(bucket_hash, bucket_list);
tommy_list_sort(bucket_list, bucket_cmp);
block_off_t count = 0;
for (tommy_node* i = tommy_list_head(bucket_list); i != 0; i = i->next) {
struct snapraid_bucket* entry = i->data;
count += entry->count_scrubbed + entry->count_justsynced;
log_tag("content_info:bucket:%" PRIu64 ":%u:%u\n", (uint64_t)entry->time_at, entry->count_scrubbed, entry->count_justsynced);
}
*bucketcount = count;
}
/****************************************************************************/
/* format */

21
cmdline/elem.h
View file

@ -1233,10 +1233,27 @@ static inline snapraid_info info_get(tommy_arrayblkof* array, block_off_t pos)
return info;
}
/****************************************************************************/
/* bucket */
struct snapraid_bucket {
time_t time_at; /**< Time of the scrub */
block_off_t count_scrubbed; /**< Number of blocks scrubbed */
block_off_t count_justsynced; /**< Number of blocks justsynced */
tommy_node node;
};
void bucket_free(struct snapraid_bucket* bucket);
/**
* Compare times
* Insert an bucket record into the hashtable
*/
int time_compare(const void* void_a, const void* void_b);
void bucket_insert(tommy_hashdyn* bucket_hash, time_t time_at, block_off_t count, int justsynced);
/**
* Convert the bucket hashtable to a sorted list
*/
void bucket_to_list(tommy_hashdyn* bucket_hash, tommy_list* bucket_list, block_off_t* bucketcount);
/****************************************************************************/
/* format */

78
cmdline/scrub.c
View file

@ -783,13 +783,11 @@ int state_scrub(struct snapraid_state* state, int plan, int olderthan)
{
block_off_t blockmax;
block_off_t countlimit;
block_off_t i;
block_off_t count;
time_t recentlimit;
int ret;
struct snapraid_parity_handle parity_handle[LEV_MAX];
struct snapraid_plan ps;
time_t* timemap;
unsigned process_error;
time_t now;
unsigned l;
@ -859,22 +857,10 @@ int state_scrub(struct snapraid_state* state, int plan, int olderthan)
}
}
/* identify the time limit */
/* we sort all the block times, and we identify the time limit for which we reach the quota */
/* this allow to process first the oldest blocks */
timemap = malloc_nofail(blockmax * sizeof(time_t));
/* copy the info in the temp vector */
count = 0;
log_tag("block_count:%u\n", blockmax);
for (i = 0; i < blockmax; ++i) {
snapraid_info info = info_get(&state->infoarr, i);
/* skip unused blocks */
if (info == 0)
continue;
timemap[count++] = info_get_time(info);
for (tommy_node* j = tommy_list_head(&state->bucketlist); j != 0; j = j->next) {
struct snapraid_bucket* bucket = j->data;
count += bucket->count_scrubbed + bucket->count_justsynced;
}
if (!count) {
@ -884,42 +870,41 @@ int state_scrub(struct snapraid_state* state, int plan, int olderthan)
/* LCOV_EXCL_STOP */
}
/* sort it */
qsort(timemap, count, sizeof(time_t), time_compare);
/* output the info map */
i = 0;
log_tag("info_count:%u\n", count);
while (i < count) {
unsigned j = i + 1;
while (j < count && timemap[i] == timemap[j])
++j;
log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)timemap[i], j - i);
i = j;
}
/* compute the limits from count/recentlimit */
if (ps.plan == SCRUB_AUTO) {
/* no more than the full count */
if (countlimit > count)
countlimit = count;
/* decrease until we reach the specific recentlimit */
while (countlimit > 0 && timemap[countlimit - 1] > recentlimit)
--countlimit;
/* by default process everything */
ps.timelimit = now;
ps.lastlimit = 0;
/* if there is something to scrub */
if (countlimit > 0) {
/* get the most recent time we want to scrub */
ps.timelimit = timemap[countlimit - 1];
tommy_node* j = tommy_list_head(&state->bucketlist);
block_off_t processed_count = 0;
while (j) {
struct snapraid_bucket* bucket = j->data;
block_off_t bucket_count = bucket->count_justsynced + bucket->count_scrubbed;
/* count how many entries for this exact time we have to scrub */
/* if the blocks have all the same time, we end with countlimit == lastlimit */
ps.lastlimit = 1;
while (countlimit > ps.lastlimit && timemap[countlimit - ps.lastlimit - 1] == ps.timelimit)
++ps.lastlimit;
} else {
/* if nothing to scrub, disable also other limits */
if (bucket->time_at > recentlimit) {
ps.timelimit = recentlimit;
ps.lastlimit = 0;
break;
}
if (processed_count + bucket_count > countlimit) {
ps.timelimit = bucket->time_at;
ps.lastlimit = countlimit - processed_count;
processed_count = countlimit;
break;
}
processed_count += bucket_count;
j = j->next;
}
/* if nothing to scrub, disable also other limits */
if (processed_count == 0) {
ps.timelimit = 0;
ps.lastlimit = 0;
}
@ -933,9 +918,6 @@ int state_scrub(struct snapraid_state* state, int plan, int olderthan)
ps.lastlimit = 0;
}
/* free the temp vector */
free(timemap);
/* open the file for reading */
for (l = 0; l < state->level; ++l) {
ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);

36
cmdline/state.c
View file

@ -181,7 +181,10 @@ void state_init(struct snapraid_state* state)
state->no_conf = 0;
for (i = 0; i < SMART_IGNORE_MAX; ++i)
state->smartignore[i] = 0;
state->rehash_blocks = 0;
state->bad_blocks = 0;
state->unsynced_blocks = 0;
state->unscrubbed_blocks = 0;
state->thermal_stop_gathering = 0;
state->thermal_ambient_temperature = 0;
state->thermal_highest_temperature = 0;
@ -200,6 +203,7 @@ void state_init(struct snapraid_state* state)
tommy_hashdyn_init(&state->previmportset);
tommy_hashdyn_init(&state->searchset);
tommy_arrayblkof_init(&state->infoarr, sizeof(snapraid_info));
tommy_list_init(&state->bucketlist);
}
void state_done(struct snapraid_state* state)
@ -215,6 +219,7 @@ void state_done(struct snapraid_state* state)
tommy_hashdyn_done(&state->previmportset);
tommy_hashdyn_done(&state->searchset);
tommy_arrayblkof_done(&state->infoarr);
tommy_list_foreach(&state->bucketlist, (tommy_foreach_func*)bucket_free);
}
/**
@ -1961,6 +1966,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
int ret;
tommy_array disk_mapping;
uint32_t mapping_max;
tommy_hashdyn bucket_hash;
blockmax = 0;
count_file = 0;
@ -1974,6 +1980,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
crc_checked = 0;
mapping_max = 0;
tommy_array_init(&disk_mapping);
tommy_hashdyn_init(&bucket_hash);
ret = sread(f, buffer, 12);
if (ret < 0) {
@ -2325,6 +2332,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
}
info = info_make(t64 + v_oldest, bad, rehash, justsynced);
bucket_insert(&bucket_hash, t64 + v_oldest, v_count, justsynced);
} else {
info = 0;
}
@ -3129,8 +3138,15 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
log_tag("content_info:block_unsynced:%u\n", count_unsynced);
log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed);
/* store the unsynced_blocks */
/* store the blocks counters */
state->rehash_blocks = count_rehash;
state->bad_blocks = count_bad;
state->unsynced_blocks = count_unsynced;
state->unscrubbed_blocks = count_unscrubbed;
/* store the bucket info list */
bucket_to_list(&bucket_hash, &state->bucketlist, &state->bucketcount);
tommy_hashdyn_done(&bucket_hash);
}
struct state_write_thread_context {
@ -3145,7 +3161,8 @@ struct state_write_thread_context {
int info_has_rehash;
STREAM* f;
int first;
/* output */
/* output (required to postpone the output to the terminal after the latest write) */
uint32_t crc;
unsigned count_file;
unsigned count_hardlink;
@ -3180,6 +3197,7 @@ static void* state_write_thread(void* arg)
block_off_t idx;
block_off_t begin;
unsigned l, s;
tommy_hashdyn bucket_hash;
count_file = 0;
count_hardlink = 0;
@ -3189,6 +3207,7 @@ static void* state_write_thread(void* arg)
count_rehash = 0;
count_unsynced = 0;
count_unscrubbed = 0;
tommy_hashdyn_init(&bucket_hash);
/* force version 3 a we want to always store the parity size */
/* write header */
@ -3593,6 +3612,9 @@ static void* state_write_thread(void* arg)
t = info_get_time(info);
if (context->first)
bucket_insert(&bucket_hash, t, count, flag & 8);
/* truncate any time that is in the future */
if (t > info_now)
t = info_now;
@ -3665,6 +3687,11 @@ static void* state_write_thread(void* arg)
context->count_unsynced = count_unsynced;
context->count_unscrubbed = count_unscrubbed;
/* store the bucket info list */
if (context->first)
bucket_to_list(&bucket_hash, &state->bucketlist, &state->bucketcount);
tommy_hashdyn_done(&bucket_hash);
return 0;
}
@ -4020,8 +4047,11 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
log_tag("content_info:block_unsynced:%u\n", count_unsynced);
log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed);
/* store the unsynced_blocks */
/* store the blocks counters */
state->rehash_blocks = count_rehash;
state->bad_blocks = count_bad;
state->unsynced_blocks = count_unsynced;
state->unscrubbed_blocks = count_unscrubbed;
*out_crc = crc;
}

5
cmdline/state.h
View file

@ -142,7 +142,10 @@ struct snapraid_state {
unsigned besthash; /**< Best hash suggested. */
const char* command; /**< Command running. */
int smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for all devices. */
uint32_t rehash_blocks; /**< Blocks marked rehash */
uint32_t bad_blocks; /**< Blocks marked bad */
uint32_t unsynced_blocks; /**< If the parity is invalid, and a sync is needed. */
uint32_t unscrubbed_blocks; /**< Blocks never scrubbed */
tommy_list contentlist; /**< List of content files. */
tommy_list disklist; /**< List of all the disks. */
@ -153,6 +156,8 @@ struct snapraid_state {
tommy_hashdyn previmportset; /**< Hashtable by prevhash of all the import blocks. Valid only if we are in a rehash state. */
tommy_hashdyn searchset; /**< Hashtable by timestamp of all the search files. */
tommy_arrayblkof infoarr; /**< Block information array. */
tommy_list bucketlist; /**< Sorted list of info bucket. Derived from infoarr, but updated only state_read()/state_write() */
block_off_t bucketcount; /**< Sum of all the counts of the list */
/* Thermal */
tommy_list thermallist; /**< List of all thermal state */

229
cmdline/status.c
View file

@ -48,18 +48,12 @@ int state_status(struct snapraid_state* state)
{
block_off_t blockmax;
block_off_t i;
time_t* timemap;
time_t now;
block_off_t bad;
block_off_t bad_first;
block_off_t bad_last;
block_off_t rehash;
block_off_t count;
unsigned l;
unsigned dayoldest, daymedian, daynewest;
unsigned bar_scrubbed[GRAPH_COLUMN];
unsigned bar_new[GRAPH_COLUMN];
unsigned barpos;
unsigned barmax;
time_t oldest, newest, median;
unsigned x, y;
@ -72,7 +66,6 @@ int state_status(struct snapraid_state* state)
uint64_t file_block_count;
uint64_t file_block_free;
block_off_t parity_block_free;
unsigned unscrubbed_blocks;
uint64_t all_wasted;
int free_not_zero;
@ -289,100 +282,28 @@ int state_status(struct snapraid_state* state)
log_tag("summary:total_use_percent:%u\n", muldiv(file_block_count, 100, file_block_count + file_block_free));
log_flush();
/* copy the info a temp vector, and count bad/rehash/unsynced blocks */
timemap = malloc_nofail(blockmax * sizeof(time_t));
bad = 0;
bad_first = 0;
bad_last = 0;
oldest = 0;
median = 0;
newest = 0;
count = 0;
rehash = 0;
unscrubbed_blocks = 0;
log_tag("block_count:%u\n", blockmax);
for (i = 0; i < blockmax; ++i) {
int one_invalid;
int one_valid;
for (tommy_node* j = tommy_list_head(&state->bucketlist); j != 0; j = j->next) {
struct snapraid_bucket* bucket = j->data;
block_off_t bucket_count = bucket->count_scrubbed + bucket->count_justsynced;
snapraid_info info = info_get(&state->infoarr, i);
if (count == 0)
oldest = bucket->time_at;
if (count < state->bucketcount / 2)
median = bucket->time_at;
newest = bucket->time_at;
/* for each disk */
one_invalid = 0;
one_valid = 0;
if (state->opt.gui_verbose) { /* avoid this time consuming operation if not needed */
for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
struct snapraid_block* block = fs_par2block_find(disk, i);
if (block_has_file(block))
one_valid = 1;
if (block_has_invalid_parity(block))
one_invalid = 1;
}
}
/* skip unused blocks */
if (info != 0) {
time_t scrub_time;
if (info_get_bad(info)) {
if (bad == 0)
bad_first = i;
bad_last = i;
++bad;
}
if (info_get_rehash(info))
++rehash;
scrub_time = info_get_time(info);
if (info_get_justsynced(info)) {
++unscrubbed_blocks;
/* mark the time as not scrubbed */
scrub_time |= TIME_NEW;
}
timemap[count++] = scrub_time;
}
if (state->opt.gui_verbose) {
if (info != 0)
log_tag("block:%u:%" PRIu64 ":%s:%s:%s:%s\n", i, (uint64_t)info_get_time(info), one_valid ? "used" : "", one_invalid ? "unsynced" : "", info_get_bad(info) ? "bad" : "", info_get_rehash(info) ? "rehash" : "");
else
log_tag("block_noinfo:%u:%s:%s\n", i, one_valid ? "used" : "", one_invalid ? "unsynced" : "");
}
count += bucket_count;
}
log_flush();
if (!count) {
log_fatal(EUSER, "The array is empty.\n");
free(timemap);
return 0;
}
/* sort the info to get the time info */
sort_time(timemap, count);
/* output the info map */
i = 0;
log_tag("info_count:%u\n", count);
while (i < count) {
unsigned j = i + 1;
while (j < count && timemap[i] == timemap[j])
++j;
if ((timemap[i] & TIME_NEW) == 0) {
log_tag("info_time:%" PRIu64 ":%u:scrubbed\n", (uint64_t)timemap[i], j - i);
} else {
log_tag("info_time:%" PRIu64 ":%u:new\n", (uint64_t)(timemap[i] & ~TIME_NEW), j - i);
}
i = j;
}
oldest = timemap[0];
median = timemap[count / 2];
newest = timemap[count - 1];
dayoldest = day_ago(oldest, now);
daymedian = day_ago(median, now);
daynewest = day_ago(newest, now);
@ -392,29 +313,19 @@ int state_status(struct snapraid_state* state)
log_tag("summary:scrub_newest_days:%u\n", daynewest);
/* compute graph limits */
barpos = 0;
barmax = 0;
for (i = 0; i < GRAPH_COLUMN; ++i) {
time_t limit;
unsigned step_scrubbed, step_new;
memset(bar_scrubbed, 0, sizeof(bar_scrubbed));
memset(bar_new, 0, sizeof(bar_new));
for (tommy_node* j = tommy_list_head(&state->bucketlist); j != 0; j = j->next) {
struct snapraid_bucket* bucket = j->data;
limit = oldest + (newest - oldest) * (i + 1) / GRAPH_COLUMN;
unsigned column = muldiv(bucket->time_at - oldest, GRAPH_COLUMN, newest - oldest + 1);
step_scrubbed = 0;
step_new = 0;
while (barpos < count && timemap[barpos] <= limit) {
if ((timemap[barpos] & TIME_NEW) != 0)
++step_new;
else
++step_scrubbed;
++barpos;
}
bar_scrubbed[column] += bucket->count_scrubbed;
bar_new[column] += bucket->count_justsynced;
if (step_new + step_scrubbed > barmax)
barmax = step_new + step_scrubbed;
bar_scrubbed[i] = step_scrubbed;
bar_new[i] = step_new;
if (bar_scrubbed[column] + bar_new[column] > barmax)
barmax = bar_scrubbed[column] + bar_new[column];
}
/* output scrub history as structured data */
@ -480,8 +391,8 @@ int state_status(struct snapraid_state* state)
printf("No sync is in progress.\n");
}
if (unscrubbed_blocks) {
printf("%u%% of the array is not scrubbed.\n", muldiv_upper(unscrubbed_blocks, 100, blockmax));
if (state->unscrubbed_blocks) {
printf("%u%% of the array is not scrubbed.\n", muldiv_upper(state->unscrubbed_blocks, 100, blockmax));
} else {
printf("The full array was scrubbed at least one time.\n");
}
@ -493,8 +404,8 @@ int state_status(struct snapraid_state* state)
printf("No file has a zero sub-second timestamp.\n");
}
if (rehash) {
printf("You have a rehash in progress at %u%%.\n", muldiv(count - rehash, 100, count));
if (state->rehash_blocks) {
printf("You have a rehash in progress at %u%%.\n", muldiv(count - state->rehash_blocks, 100, count));
} else {
if (state->besthash != state->hash) {
printf("No rehash is in progress, but for optimal performance one is recommended.\n");
@ -503,60 +414,53 @@ int state_status(struct snapraid_state* state)
}
}
if (bad) {
printf("DANGER! In the array there are %u errors!\n\n", bad);
if (state->bad_blocks) {
printf("DANGER! In the array there are %u errors!\n\n", state->bad_blocks);
if (bad_last - bad_first + 1 == bad) {
printf("They are from block %u to %u.\n", bad_first, bad_last);
} else {
block_off_t bad_range;
block_off_t bad_count;
block_off_t range_start;
block_off_t range_count;
block_off_t bad_range;
block_off_t bad_count;
block_off_t range_start;
block_off_t range_count;
printf("They are from block %u to %u, specifically at blocks:", bad_first, bad_last);
printf("They are at blocks:");
/* print some of the errors */
bad_range = 0;
bad_count = 0;
range_start = 0;
range_count = 0;
for (i = 0; i <= blockmax; ++i) { /* one extra iteration to print the final range */
snapraid_info info = 0;
int is_bad = 0;
/* print some of the errors */
bad_range = 0;
bad_count = 0;
range_start = 0;
range_count = 0;
for (i = 0; i <= blockmax; ++i) { /* one extra iteration to print the final range */
snapraid_info info = 0;
int is_bad = 0;
if (i < blockmax) {
info = info_get(&state->infoarr, i);
if (info != 0) /* unused blocks are never bad */
is_bad = info_get_bad(info);
}
if (is_bad) {
/* create or extend the range */
if (!range_count)
range_start = i;
++range_count;
} else {
/* break the range */
if (range_count) {
if (range_count == 1) {
printf(" %u", range_start);
} else {
printf(" %u-%u", range_start, range_start + range_count - 1);
}
bad_count += range_count;
++bad_range;
range_count = 0;
if (i < blockmax) {
info = info_get(&state->infoarr, i);
if (info != 0) /* unused blocks are never bad */
is_bad = info_get_bad(info);
}
if (is_bad) {
/* create or extend the range */
if (!range_count)
range_start = i;
++range_count;
} else {
/* break the range */
if (range_count) {
if (range_count == 1) {
printf(" %u", range_start);
} else {
printf(" %u-%u", range_start, range_start + range_count - 1);
}
}
if (bad_range > 100) {
printf(" and %u more...", bad - bad_count);
break;
bad_count += range_count;
++bad_range;
range_count = 0;
}
}
printf("\n");
if (bad_range > 100) {
printf(" and %u more...", state->bad_blocks - bad_count);
break;
}
}
printf("\n");
@ -567,16 +471,13 @@ int state_status(struct snapraid_state* state)
printf("No error detected.\n");
}
if (bad)
if (state->bad_blocks)
log_tag("summary:exit:bad\n");
else if (state->unsynced_blocks != 0)
log_tag("summary:exit:unsynced\n");
else
log_tag("summary:exit:ok\n");
/* free the temp vector */
free(timemap);
return 0;
}

96
cmdline/util.c
View file

@ -854,99 +854,3 @@ unsigned muldiv_upper(uint64_t v, uint64_t mul, uint64_t div)
return (uint32_t)((v * mul + div - 1) / div);
}
/****************************************************************************/
/* sort */
/**
* Three-way partition for quicksort
* Partitions array into three parts: < pivot, == pivot, > pivot
* Returns the boundaries of the equal section via lt and gt pointers
*/
static void partition3way(time_t *arr, size_t left, size_t right, size_t *lt, size_t *gt)
{
/* choose median-of-three as pivot to avoid worst case on sorted data */
size_t mid = left + (right - left) / 2;
time_t pivot;
/* sort left, mid, right and use median as pivot */
if (arr[left] > arr[mid]) {
uint64_t tmp = arr[left]; arr[left] = arr[mid]; arr[mid] = tmp;
}
if (arr[mid] > arr[right]) {
uint64_t tmp = arr[mid]; arr[mid] = arr[right]; arr[right] = tmp;
if (arr[left] > arr[mid]) {
tmp = arr[left]; arr[left] = arr[mid]; arr[mid] = tmp;
}
}
pivot = arr[mid];
// Dutch National Flag partition
size_t i = left;
size_t lt_idx = left;
size_t gt_idx = right;
while (i <= gt_idx) {
if (arr[i] < pivot) {
uint64_t tmp = arr[lt_idx];
arr[lt_idx] = arr[i];
arr[i] = tmp;
lt_idx++;
i++;
} else if (arr[i] > pivot) {
uint64_t tmp = arr[i];
arr[i] = arr[gt_idx];
arr[gt_idx] = tmp;
gt_idx--;
} else {
i++;
}
}
*lt = lt_idx;
*gt = gt_idx;
}
/**
* Recursive three-way quicksort
*/
static void quicksort3way_recursive(time_t *arr, size_t left, size_t right)
{
if (left >= right)
return;
/* use insertion sort for small subarrays (optimization) */
if (right - left < 16) {
for (size_t i = left + 1; i <= right; i++) {
time_t key = arr[i];
size_t j = i;
while (j > left && arr[j - 1] > key) {
arr[j] = arr[j - 1];
j--;
}
arr[j] = key;
}
return;
}
size_t lt, gt;
partition3way(arr, left, right, &lt, &gt);
/*
* Recursively sort elements less than and greater than pivot
* Elements equal to pivot are already in correct position
*/
if (lt > left) {
quicksort3way_recursive(arr, left, lt - 1);
}
if (gt < right) {
quicksort3way_recursive(arr, gt + 1, right);
}
}
void sort_time(time_t *arr, size_t n)
{
if (n <= 1)
return;
quicksort3way_recursive(arr, 0, n - 1);
}

12
cmdline/util.h
View file

@ -273,17 +273,5 @@ static inline int bit_vect_test(bit_vect_t* bit_vect, size_t off)
unsigned muldiv(uint64_t v, uint64_t mul, uint64_t div);
unsigned muldiv_upper(uint64_t v, uint64_t mul, uint64_t div);
/****************************************************************************/
/* sort */
/**
* Sort an array of uint64_t integers in-place
* Optimized for arrays with many duplicate values
*
* @param arr Pointer to array of uint64_t
* @param n Number of elements in the array
*/
void sort_time(time_t* arr, size_t n);
#endif

103
doc/snapraid_log.1
View file

@ -31,7 +31,7 @@ NAME:VALUE:VALUE:VALUE...<EOL>
.TP
.B NAME
This is the primary tag identifier, always a lowercase string
without spaces (e.g., \`summary\`, \`block\`, \`info_time\`).
without spaces (e.g., \`summary\`, \`block\`).
.TP
.B VALUE
Following the \`NAME\`, these can be additional sub\-identifiers,
@ -338,6 +338,27 @@ meaning their parity data is out of date.
The total counter of blocks.
.PD
.RE
.TP
.B content_info:bucket_count:<uint>
Total number of info records about past scrub and sync
operations stored in the content file.
.TP
.B content_info:bucket:<time>:<count_scrubbed>:<count_new>
All the info records.
.RS 4
.PD 0
.HP 4
.I <time>
Unix timestamp of the scrub/sync.
.HP 4
.I <count_scrubbed>
Number of blocks scrubbed at this timestamp.
.HP 4
.I <count_new>
Number of blocks that were synced at this
timestamp and haven\'t been scrubbed yet.
.PD
.RE
.SH DIAGNOSTICS TAGS
.TP
.B version:<version>
@ -954,89 +975,19 @@ These tags list files with zero sub\-second timestamps.
.B zerosubsecond:<disk_name>:<file_path>
Lists files that have a zero sub\-second timestamp.
Up to 50 are logged per disk.
.SS Block Information Tags
Status of all the blocks of the array.
.TP
.B block_count:<uint>
The total number of blocks in the array.
.TP
.B block:<index>:<time>:<used>:<unsynced>:<bad>:<rehash>
Status of a specific block.
Only produced when \`\-\-gui\-verbose\` option is used.
.RS 4
.PD 0
.HP 4
.I <index>
Block index.
.HP 4
.I <time>
Last scrub time (Unix timestamp). A value of 0 means
the block has never been scrubbed.
.HP 4
.I <used>
\`used\` if the block contains data, otherwise empty.
.HP 4
.I <unsynced>
\`unsynced\` if the block needs a sync, otherwise empty.
.HP 4
.I <bad>
\`bad\` if the block has errors, otherwise empty.
.HP 4
.I <rehash>
\`rehash\` if the block needs a rehash, otherwise empty.
.PD
.RE
.TP
.B block_noinfo:<index>:<used>:<unsynced>
Status of an unused block (no info recorded) in the content file.
Only produced when \`\-\-gui\-verbose\` option is used.
.RS 4
.PD 0
.HP 4
.I <index>
Block index.
.HP 4
.I <used>
\`used\` if the block contains data, otherwise empty.
.HP 4
.I <unsynced>
\`unsynced\` if the block needs a sync, otherwise empty.
.PD
.RE
.SS Scrub History Tags
These tags provide details on the scrub/sync time distribution.
.TP
.B info_count:<uint>
Total number of info records about past scrub and sync
operations stored in the content file.
.TP
.B info_time:<time>:<count>:<status>
All the info records.
.RS 4
.PD 0
.HP 4
.I <time>
Unix timestamp of the scrub/sync.
.HP 4
.I <count>
Number of blocks processed at this timestamp.
.HP 4
.I <status>
\`scrubbed\` (for blocks verified to be correct) or
\`new\` (for synced blocks that haven\'t been scrubbed yet).
.PD
.RE
.SS Scrub Graph Tags
These tags provide a simple graph data of the past scrub and sync
operations.
.TP
.B scrub_graph_range:<max_columns>:<max_height>
Dimensions of the scrub history graph obtained from the
\`info_time\` records. This provides the boundary for the
\`content_info:bucket\` records. This provides the boundary for the
precomputed graph data available in the \`scrub_graph_bar\` tags.
.RS 4
.PD 0
.HP 4
.I <max_columns>
The number of columns in the graph (constant
GRAPH_COLUMN).
The number of columns in the graph.
.HP 4
.I <max_height>
The maximum height (peak count) of blocks found

62
doc/snapraid_log.d
View file

@ -25,7 +25,7 @@ Description
=NAME
This is the primary tag identifier, always a lowercase string
without spaces (e.g., `summary`, `block`, `info_time`).
without spaces (e.g., `summary`, `block`).
=VALUE
Following the `NAME`, these can be additional sub-identifiers,
descriptive names (like a disk name), or numeric/string data
@ -210,6 +210,18 @@ Content Tags
meaning their parity data is out of date.
block - The total counter of blocks.
=content_info:bucket_count:<uint>
Total number of info records about past scrub and sync
operations stored in the content file.
=content_info:bucket:<time>:<count_scrubbed>:<count_new>
All the info records.
<time> - Unix timestamp of the scrub/sync.
<count_scrubbed> - Number of blocks scrubbed at this timestamp.
<count_new> - Number of blocks that were synced at this
timestamp and haven't been scrubbed yet.
Diagnostics Tags
=version:<version>
The version of SnapRAID run.
@ -621,54 +633,16 @@ Command Status Tags
Lists files that have a zero sub-second timestamp.
Up to 50 are logged per disk.
Block Information Tags
Status of all the blocks of the array.
=block_count:<uint>
The total number of blocks in the array.
=block:<index>:<time>:<used>:<unsynced>:<bad>:<rehash>
Status of a specific block.
Only produced when `--gui-verbose` option is used.
<index> - Block index.
<time> - Last scrub time (Unix timestamp). A value of 0 means
the block has never been scrubbed.
<used> - `used` if the block contains data, otherwise empty.
<unsynced> - `unsynced` if the block needs a sync, otherwise empty.
<bad> - `bad` if the block has errors, otherwise empty.
<rehash> - `rehash` if the block needs a rehash, otherwise empty.
=block_noinfo:<index>:<used>:<unsynced>
Status of an unused block (no info recorded) in the content file.
Only produced when `--gui-verbose` option is used.
<index> - Block index.
<used> - `used` if the block contains data, otherwise empty.
<unsynced> - `unsynced` if the block needs a sync, otherwise empty.
Scrub History Tags
These tags provide details on the scrub/sync time distribution.
=info_count:<uint>
Total number of info records about past scrub and sync
operations stored in the content file.
=info_time:<time>:<count>:<status>
All the info records.
<time> - Unix timestamp of the scrub/sync.
<count> - Number of blocks processed at this timestamp.
<status> - `scrubbed` (for blocks verified to be correct) or
`new` (for synced blocks that haven't been scrubbed yet).
Scrub Graph Tags
These tags provide a simple graph data of the past scrub and sync
operations.
=scrub_graph_range:<max_columns>:<max_height>
Dimensions of the scrub history graph obtained from the
`info_time` records. This provides the boundary for the
`content_info:bucket` records. This provides the boundary for the
precomputed graph data available in the `scrub_graph_bar` tags.
<max_columns> - The number of columns in the graph (constant
GRAPH_COLUMN).
<max_columns> - The number of columns in the graph.
<max_height> - The maximum height (peak count) of blocks found
in any column (the highest sum of scrubbed and new blocks).

62
doc/snapraid_log.txt
View file

@ -29,7 +29,7 @@ NAME:VALUE:VALUE:VALUE...<EOL>
NAME
This is the primary tag identifier, always a lowercase string
without spaces (e.g., `summary`, `block`, `info_time`).
without spaces (e.g., `summary`, `block`).
VALUE
Following the `NAME`, these can be additional sub-identifiers,
descriptive names (like a disk name), or numeric/string data
@ -220,6 +220,18 @@ These tags report the information stored in the content file.
meaning their parity data is out of date.
block - The total counter of blocks.
content_info:bucket_count:<uint>
Total number of info records about past scrub and sync
operations stored in the content file.
content_info:bucket:<time>:<count_scrubbed>:<count_new>
All the info records.
<time> - Unix timestamp of the scrub/sync.
<count_scrubbed> - Number of blocks scrubbed at this timestamp.
<count_new> - Number of blocks that were synced at this
timestamp and haven't been scrubbed yet.
4 DIAGNOSTICS TAGS
==================
@ -646,54 +658,16 @@ These tags list files with zero sub-second timestamps.
Lists files that have a zero sub-second timestamp.
Up to 50 are logged per disk.
---- 8.0.4 Block Information Tags ----
Status of all the blocks of the array.
block_count:<uint>
The total number of blocks in the array.
block:<index>:<time>:<used>:<unsynced>:<bad>:<rehash>
Status of a specific block.
Only produced when `--gui-verbose` option is used.
<index> - Block index.
<time> - Last scrub time (Unix timestamp). A value of 0 means
the block has never been scrubbed.
<used> - `used` if the block contains data, otherwise empty.
<unsynced> - `unsynced` if the block needs a sync, otherwise empty.
<bad> - `bad` if the block has errors, otherwise empty.
<rehash> - `rehash` if the block needs a rehash, otherwise empty.
block_noinfo:<index>:<used>:<unsynced>
Status of an unused block (no info recorded) in the content file.
Only produced when `--gui-verbose` option is used.
<index> - Block index.
<used> - `used` if the block contains data, otherwise empty.
<unsynced> - `unsynced` if the block needs a sync, otherwise empty.
---- 8.0.5 Scrub History Tags ----
These tags provide details on the scrub/sync time distribution.
info_count:<uint>
Total number of info records about past scrub and sync
operations stored in the content file.
info_time:<time>:<count>:<status>
All the info records.
<time> - Unix timestamp of the scrub/sync.
<count> - Number of blocks processed at this timestamp.
<status> - `scrubbed` (for blocks verified to be correct) or
`new` (for synced blocks that haven't been scrubbed yet).
---- 8.0.4 Scrub Graph Tags ----
These tags provide a simple graph data of the past scrub and sync
operations.
scrub_graph_range:<max_columns>:<max_height>
Dimensions of the scrub history graph obtained from the
`info_time` records. This provides the boundary for the
`content_info:bucket` records. This provides the boundary for the
precomputed graph data available in the `scrub_graph_bar` tags.
<max_columns> - The number of columns in the graph (constant
GRAPH_COLUMN).
<max_columns> - The number of columns in the graph.
<max_height> - The maximum height (peak count) of blocks found
in any column (the highest sum of scrubbed and new blocks).

58
tommyds/tommyhashdyn.c
View file

@ -222,3 +222,61 @@ TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn)
+ tommy_hashdyn_count(hashdyn) * (tommy_size_t)sizeof(tommy_hashdyn_node);
}
/**
* \brief Transfers all elements from the dynamic hashtable into a tommy_list.
*
* Removes every element from the \p hashdyn hashtable and inserts them
* into the provided \p list (at the tail), preserving the per-bucket order
* but not guaranteeing any particular global order.
*
* After the call:
* - the hashtable is left completely empty (\c count = 0, all buckets = NULL)
* - the target list contains all the elements that were previously in the hashtable
*
* This function is useful when you need to:
* - extract all elements to process/sort them outside the hash table
* - convert the hashtable into a list for sequential iteration
* - prepare for a full clear + re-insertion with different hash/ordering
* - move ownership of the nodes to a list-based container
*
* \note The operation is O(n) where n is the number of elements.
* \note No memory allocation is performed.
* \note The relative order of elements that were in the same bucket is preserved,
* but the order among different buckets is bucket-order dependent.
*
* Typical usage pattern:
* \code
* tommy_list all_elements;
* tommy_list_init(&all_elements);
*
* // move everything out of the hashtable into the list
* tommy_hashdyn_to_list(&hashdyn, &all_elements);
*
* // now you can sort, filter, process sequentially, etc.
* tommy_list_sort(&all_elements, compare_by_value);
* \endcode
*
* \param hashdyn The dynamic hashtable to drain
* \param list The list that will receive all the elements
*/
TOMMY_API void tommy_hashdyn_to_list(tommy_hashdyn* hashdyn, tommy_list* list)
{
tommy_size_t bucket_max = hashdyn->bucket_max;
tommy_hashdyn_node** bucket = hashdyn->bucket;
tommy_size_t pos;
/* move everything to the list */
for (pos = 0; pos < bucket_max; ++pos) {
tommy_hashdyn_node* node = bucket[pos];
while (node) {
tommy_node* node_next = node->next;
tommy_list_insert_tail(list, node, node->data);
node = node_next;
}
}
/* clear all */
memset(hashdyn->bucket, 0, hashdyn->bucket_max * sizeof(tommy_hashdyn_node*));
hashdyn->count = 0;
}

39
tommyds/tommyhashdyn.h
View file

@ -292,4 +292,43 @@ tommy_inline tommy_size_t tommy_hashdyn_count(tommy_hashdyn* hashdyn)
*/
TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn);
/**
* \brief Transfers all elements from the hashtable into a tommy_list.
*
* Removes every element from the \p hashdyn hashtable and inserts them
* into the provided \p list (at the tail), preserving the per-bucket order
* but not guaranteeing any particular global order.
*
* After the call:
* - the hashtable is left completely empty
* - the target list contains all the elements that were previously in the hashtable
*
* This function is useful when you need to:
* - extract all elements to process/sort them outside the hash table
* - convert the hashtable into a list for sequential iteration
* - prepare for a full clear + re-insertion with different hash/ordering
* - move ownership of the nodes to a list-based container
*
* \note The operation is O(n) where n is the number of elements.
* \note No memory allocation is performed.
* \note The relative order of elements that were in the same bucket is preserved,
* but the order among different buckets is bucket-order dependent.
*
* Typical usage pattern:
* \code
* tommy_list all_elements;
* tommy_list_init(&all_elements);
*
* // move everything out of the hashtable into the list
* tommy_hashdyn_to_list(&hashdyn, &all_elements);
*
* // now you can sort, filter, process sequentially, etc.
* tommy_list_sort(&all_elements, compare_by_value);
* \endcode
*
* \param hashdyn The dynamic hashtable to drain
* \param list The list that will receive all the elements
*/
TOMMY_API void tommy_hashdyn_to_list(tommy_hashdyn* hashdyn, tommy_list* list);
#endif