Log the content file counters

This commit is contained in:
Andrea Mazzoleni 2026-01-16 00:42:24 +01:00
parent e9756461af
commit 9d383f08c9
4 changed files with 172 additions and 44 deletions

View file

@ -1895,6 +1895,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
unsigned count_hardlink;
unsigned count_symlink;
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
int crc_checked;
char buffer[PATH_MAX];
int ret;
@ -1906,6 +1908,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
count_hardlink = 0;
count_symlink = 0;
count_dir = 0;
count_bad = 0;
count_rehash = 0;
crc_checked = 0;
mapping_max = 0;
tommy_array_init(&disk_mapping);
@ -2244,6 +2248,11 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
rehash = (flag & 4) != 0;
justsynced = (flag & 8) != 0;
if (bad)
++count_bad;
if (rehash)
++count_rehash;
if (rehash && state->prevhash == HASH_UNDEFINED) {
/* LCOV_EXCL_START */
decoding_error(path, f);
@ -2722,7 +2731,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* LCOV_EXCL_STOP */
}
/* log the info read from the content file */
log_tag("content_data:%s:%s\n",
buffer,
uuid);
@ -2828,7 +2836,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
state->parity[v_level].total_blocks = v_total_blocks;
state->parity[v_level].free_blocks = v_free_blocks;
/* log the info read from the content file */
log_tag("content_parity:%s:%s:%s:%" PRIi64 "\n",
lev_config_name(v_level),
state->parity[v_level].split_map[0].uuid,
@ -2959,7 +2966,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
state->parity[v_level].split_map[s].size = v_size;
/* log the info read from the content file */
if (s == 0)
pathcpy(parity_name, sizeof(parity_name), lev_config_name(v_level));
else
@ -3045,6 +3051,14 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
msg_verbose("%8u hardlinks\n", count_hardlink);
msg_verbose("%8u symlinks\n", count_symlink);
msg_verbose("%8u empty dirs\n", count_dir);
log_tag("content_info:file:%u\n", count_file);
log_tag("content_info:hardlink:%u\n", count_hardlink);
log_tag("content_info:symlink:%u\n", count_symlink);
log_tag("content_info:dir_empty:%u\n", count_dir);
log_tag("content_info:block_bad:%u\n", count_bad);
log_tag("content_info:block_rehash:%u\n", count_rehash);
log_tag("content_info:block:%u\n", blockmax);
}
struct state_write_thread_context {
@ -3058,12 +3072,15 @@ struct state_write_thread_context {
time_t info_now;
int info_has_rehash;
STREAM* f;
int first;
/* output */
uint32_t crc;
unsigned count_file;
unsigned count_hardlink;
unsigned count_symlink;
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
};
static void* state_write_thread(void* arg)
@ -3081,6 +3098,8 @@ static void* state_write_thread(void* arg)
unsigned count_hardlink;
unsigned count_symlink;
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
tommy_node* i;
block_off_t idx;
block_off_t begin;
@ -3090,6 +3109,8 @@ static void* state_write_thread(void* arg)
count_hardlink = 0;
count_symlink = 0;
count_dir = 0;
count_bad = 0;
count_rehash = 0;
/* force version 3 a we want to always store the parity size */
/* write header */
@ -3182,6 +3203,15 @@ static void* state_write_thread(void* arg)
sputb32(map->total_blocks, f);
sputb32(map->free_blocks, f);
sputbs(map->uuid, f);
if (context->first) {
log_tag("content_allocation:%s:%" PRIi64 ":%" PRIi64 "\n",
map->name,
map->total_blocks * (uint64_t)state->block_size,
map->free_blocks * (uint64_t)state->block_size);
log_tag("content_data:%s:%s\n",
map->name,
map->uuid);
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -3197,11 +3227,29 @@ static void* state_write_thread(void* arg)
sputb32(l, f);
sputb32(state->parity[l].total_blocks, f);
sputb32(state->parity[l].free_blocks, f);
if (context->first) {
log_tag("content_allocation:%s:%" PRIi64 ":%" PRIi64 "\n",
lev_config_name(l),
state->parity[l].total_blocks * (uint64_t)state->block_size,
state->parity[l].free_blocks * (uint64_t)state->block_size);
}
sputb32(state->parity[l].split_mac, f);
for (s = 0; s < state->parity[l].split_mac; ++s) {
char parity_name[64];
sputbs(state->parity[l].split_map[s].path, f);
sputbs(state->parity[l].split_map[s].uuid, f);
sputb64(state->parity[l].split_map[s].size, f);
if (s == 0)
pathcpy(parity_name, sizeof(parity_name), lev_config_name(l));
else
pathprint(parity_name, sizeof(parity_name), "%s/%u", lev_config_name(l), s);
if (context->first) {
log_tag("content_parity:%s:%s:%s:%" PRIi64 "\n",
parity_name,
state->parity[l].split_map[s].uuid,
state->parity[l].split_map[s].path,
state->parity[l].split_map[s].size);
}
}
if (serror(f)) {
/* LCOV_EXCL_START */
@ -3440,10 +3488,14 @@ static void* state_write_thread(void* arg)
if (info) {
/* other flags */
flag = 1; /* info is present */
if (info_get_bad(info))
if (info_get_bad(info)) {
flag |= 2;
if (info_get_rehash(info))
++count_bad;
}
if (info_get_rehash(info)) {
flag |= 4;
++count_rehash;
}
if (info_get_justsynced(info))
flag |= 8;
sputb32(flag, f);
@ -3517,6 +3569,8 @@ static void* state_write_thread(void* arg)
context->count_hardlink = count_hardlink;
context->count_symlink = count_symlink;
context->count_dir = count_dir;
context->count_bad = count_bad;
context->count_rehash = count_rehash;
return 0;
}
@ -3545,6 +3599,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
unsigned count_hardlink;
unsigned count_symlink;
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
/* blocks of all array */
blockmax = parity_allocated_size(state);
@ -3609,14 +3665,17 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
#if HAVE_MT_WRITE
/* start all writing threads */
first = 1;
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
struct state_write_thread_context* context;
char tmp[PATH_MAX];
char esc_buffer[ESC_MAX];
STREAM* f;
msg_progress("Saving state to %s...\n", content->content);
log_tag("content_write:%s\n", esc_tag(content->content, esc_buffer));
pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
@ -3649,6 +3708,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
context->info_now = info_now;
context->info_has_rehash = info_has_rehash;
context->f = f;
context->first = first;
first = 0;
thread_create(&context->thread, state_write_thread, context);
@ -3663,6 +3724,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
count_hardlink = 0;
count_symlink = 0;
count_dir = 0;
count_bad = 0;
count_rehash = 0;
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
@ -3710,6 +3773,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
count_hardlink = context->count_hardlink;
count_symlink = context->count_symlink;
count_dir = context->count_dir;
cound_bad = context->count_bad;
count_rehash = context->count_rehash;
} else {
if (crc != context->crc) {
/* LCOV_EXCL_START */
@ -3791,6 +3856,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
context->info_now = info_now;
context->info_has_rehash = info_has_rehash;
context->f = f;
context->first = 1;
retval = state_write_thread(context);
@ -3831,6 +3897,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
count_hardlink = context->count_hardlink;
count_symlink = context->count_symlink;
count_dir = context->count_dir;
count_bad = context->count_bad;
count_rehash = context->count_rehash;
free(context);
#endif
@ -3840,6 +3908,14 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
msg_verbose("%8u symlinks\n", count_symlink);
msg_verbose("%8u empty dirs\n", count_dir);
log_tag("content_info:file:%u\n", count_file);
log_tag("content_info:hardlink:%u\n", count_hardlink);
log_tag("content_info:symlink:%u\n", count_symlink);
log_tag("content_info:dir_empty:%u\n", count_dir);
log_tag("content_info:block_bad:%u\n", count_bad);
log_tag("content_info:block_rehash:%u\n", count_rehash);
log_tag("content_info:block:%u\n", blockmax);
*out_crc = crc;
}
@ -3860,7 +3936,8 @@ void state_read(struct snapraid_state* state)
pathcpy(path, sizeof(path), content->content);
if (!state->no_conf) {
log_tag("content:%s\n", path);
char esc_buffer[ESC_MAX];
log_tag("content:%s\n", esc_tag(path, esc_buffer));
log_flush();
}
msg_progress("Loading state from %s...\n", path);

View file

@ -71,8 +71,7 @@ Some tags are output only when the \`\-\-gui\` option is used. They provide
extra verbose state and runtime progress only useful for a GUI
interface.
.SH CONFIGURATION TAGS
These tags report the memory usage and global configuration after
scanning the disk and reading the list of files.
These tags report the memory usage and global configuration.
.TP
.B conf:file:<path>
Configuration file loaded (escaped).
@ -143,12 +142,19 @@ exclude specific files or directories from the SnapRAID array.
.B filter:nohidden:
If the no\-hidden filter is enabled, meaning hidden files/directories
(starting with \'.\') are excluded from the array.
.SH CONTENT TAGS
These tags report the information stored in the content file.
.TP
.B content:<path>
The absolute path to a content file being used (escaped).
Content files store the metadata (like file hashes and timestamps)
for the array.
.TP
.B content_write:<path>
The absolute path to a content file being written (escaped).
You will see multiple writes as the content file is saved in
multiple copies.
.TP
.B content_parity:<parity_level>[/<split_index>]:<uuid>:<path>:<size>
The parity files as stored in the content file.
.RS 4
@ -202,6 +208,20 @@ The size of the data in the disk or parity (uint64).
The free size in the disk or parity (uint64).
.PD
.RE
.TP
.B content_info:<kind>:<counter>
The counters in the content file.
.RS 4
.PD 0
.HP 4
.I <kind>
One of file, hardlink, symlink, dir_empty,
block_bad, block_rehash, block.
.HP 4
.I <counter>
The number of elements (uint)
.PD
.RE
.SH DIAGNOSTICS TAGS
.TP
.B version:<version>

View file

@ -53,8 +53,7 @@ Description
Configuration Tags
These tags report the memory usage and global configuration after
scanning the disk and reading the list of files.
These tags report the memory usage and global configuration.
=conf:file:<path>
Configuration file loaded (escaped).
@ -108,10 +107,18 @@ Configuration Tags
If the no-hidden filter is enabled, meaning hidden files/directories
(starting with '.') are excluded from the array.
Content Tags
These tags report the information stored in the content file.
=content:<path>
The absolute path to a content file being used (escaped).
Content files store the metadata (like file hashes and timestamps)
for the array.
=content_write:<path>
The absolute path to a content file being written (escaped).
You will see multiple writes as the content file is saved in
multiple copies.
=content_parity:<parity_level>[/<split_index>]:<uuid>:<path>:<size>
The parity files as stored in the content file.
@ -140,6 +147,13 @@ Configuration Tags
<size> - The size of the data in the disk or parity (uint64).
<free_size> - The free size in the disk or parity (uint64).
=content_info:<kind>:<counter>
The counters in the content file.
<kind> - One of file, hardlink, symlink, dir_empty,
block_bad, block_rehash, block.
<counter> - The number of elements (uint)
Diagnostics Tags
=version:<version>
The version of SnapRAID run.

View file

@ -59,8 +59,7 @@ interface.
2 CONFIGURATION TAGS
====================
These tags report the memory usage and global configuration after
scanning the disk and reading the list of files.
These tags report the memory usage and global configuration.
conf:file:<path>
Configuration file loaded (escaped).
@ -114,11 +113,22 @@ scanning the disk and reading the list of files.
If the no-hidden filter is enabled, meaning hidden files/directories
(starting with '.') are excluded from the array.
3 CONTENT TAGS
==============
These tags report the information stored in the content file.
content:<path>
The absolute path to a content file being used (escaped).
Content files store the metadata (like file hashes and timestamps)
for the array.
content_write:<path>
The absolute path to a content file being written (escaped).
You will see multiple writes as the content file is saved in
multiple copies.
content_parity:<parity_level>[/<split_index>]:<uuid>:<path>:<size>
The parity files as stored in the content file.
@ -146,8 +156,15 @@ scanning the disk and reading the list of files.
<size> - The size of the data in the disk or parity (uint64).
<free_size> - The free size in the disk or parity (uint64).
content_info:<kind>:<counter>
The counters in the content file.
3 DIAGNOSTICS TAGS
<kind> - One of file, hardlink, symlink, dir_empty,
block_bad, block_rehash, block.
<counter> - The number of elements (uint)
4 DIAGNOSTICS TAGS
==================
version:<version>
@ -186,7 +203,7 @@ scanning the disk and reading the list of files.
Size of the internal `snapraid_dir` structure in bytes (uint64).
4 SCAN TAGS
5 SCAN TAGS
===========
These tags report detected differences between the filesystem and the
@ -275,7 +292,7 @@ These tags provide a final count of all detected changes.
The array needs a `sync` operation.
5 GENERAL PROGRESS AND EXECUTION TAGS
6 GENERAL PROGRESS AND EXECUTION TAGS
=====================================
These tags are used to report the runtime processing of the commands
@ -331,7 +348,7 @@ Only produced when `--gui` option is used.
<msg> - Text message exactly as it's printed on screen.
6 THERMAL TAGS
7 THERMAL TAGS
==============
The following tags are produced by the thermal monitoring functions
@ -434,12 +451,12 @@ disk access.
are being spun back up to resume the operation.
7 COMMAND STATUS TAGS
8 COMMAND STATUS TAGS
=====================
This section describes the tags output with the `status` command.
---- 7.0.1 Summary Tags ----
---- 8.0.1 Summary Tags ----
These tags provide general information about the array configuration
and overall state.
@ -523,7 +540,7 @@ and overall state.
unsynced - The array has an interrupted sync.
The array needs another `sync` operation.
---- 7.0.2 Per-Disk Tags ----
---- 8.0.2 Per-Disk Tags ----
These tags provide statistics for individual data disks. <name>
is the disk name.
@ -564,14 +581,14 @@ is the disk name.
summary:disk_use_percent:<name>:<uint>
Disk use percentage relative to `disk_block_max`.
---- 7.0.3 File Timestamp Tags ----
---- 8.0.3 File Timestamp Tags ----
These tags list files with zero sub-second timestamps.
zerosubsecond:<disk_name>:<file_path>
Lists files that have a zero sub-second timestamp.
Up to 50 are logged per disk.
---- 7.0.4 Block Information Tags ----
---- 8.0.4 Block Information Tags ----
These tags are generated when the GUI option is enabled (`state->opt.gui`).
block_count:<uint>
@ -595,7 +612,7 @@ These tags are generated when the GUI option is enabled (`state->opt.gui`).
<used> - `used` if the block contains data, otherwise empty.
<unsynced> - `unsynced` if the block needs a sync, otherwise empty.
---- 7.0.5 Scrub History Tags ----
---- 8.0.5 Scrub History Tags ----
These tags provide details on the scrub/sync time distribution.
info_count:<uint>
@ -630,13 +647,13 @@ These tags provide details on the scrub/sync time distribution.
<new> - Count of newly synced (unscrubbed) blocks in that day range.
8 COMMAND SYNC/SCRUB TAGS
9 COMMAND SYNC/SCRUB TAGS
=========================
Tags specific for the `sync` and `scrub` commands.
Also the error tags are possible.
---- 8.0.1 Summary Tags ----
---- 9.0.1 Summary Tags ----
These tags provide summary statistics at the end of the command.
hash_summary:error_file:<count>
@ -678,13 +695,13 @@ These tags provide summary statistics at the end of the command.
or filesystem inspection.
9 COMMAND CHECK/FIX TAGS
========================
10 COMMAND CHECK/FIX TAGS
=========================
Tags specific for the `check` and `fix` commands.
Also the error tags are possible.
---- 9.0.1 Fixed/Recovered Tags ----
---- 10.0.1 Fixed/Recovered Tags ----
These tags log successful repairs during fix operations.
fixed:<block_pos>:<disk_name>:<file_path>:<msg>
@ -738,7 +755,7 @@ These tags log successful repairs during fix operations.
<msg> - A descriptive message indicating the nature of the fix
(e.g., `Fixed dir error`).
---- 9.0.2 Unrecoverable Error Tags ----
---- 10.0.2 Unrecoverable Error Tags ----
These tags log errors that could not be recovered.
unrecoverable:<block_pos>:<disk_name>:<file_path>:<msg>
@ -762,7 +779,7 @@ These tags log errors that could not be recovered.
<msg> - A descriptive message indicating the nature of the problem
(e.g., `Not setting modification time to avoid inode collision`).
---- 9.0.3 File Status Tags ----
---- 10.0.3 File Status Tags ----
These tags log the final status of files after processing.
status:unrecoverable:<disk_name>:<file_path>
@ -796,7 +813,7 @@ These tags log the final status of files after processing.
<disk_name> - The name of the disk.
<file_path> - The file path relative to disk root (escaped).
---- 9.0.4 Summary Tags ----
---- 10.0.4 Summary Tags ----
These tags provide summary statistics at the end of the command.
summary:error:<error_count>
@ -828,12 +845,12 @@ These tags provide summary statistics at the end of the command.
error - Indicates that errors were found (audit-only mode).
10 COMMAND LIST TAGS
11 COMMAND LIST TAGS
====================
This section describes the tags output with the `list` command.
---- 10.0.1 List Tags ----
---- 11.0.1 List Tags ----
These tags log details about individual files and links processed during
the listing operation.
@ -862,7 +879,7 @@ the listing operation.
(escaped).
<linkto> - The target path of the link (escaped).
---- 10.0.2 Summary Tags ----
---- 11.0.2 Summary Tags ----
These tags log final statistics about the entire listing operation.
summary:file_count:<count>
@ -887,12 +904,12 @@ These tags log final statistics about the entire listing operation.
and you won't get this tag.
11 COMMAND DUP TAGS
12 COMMAND DUP TAGS
===================
This section describes the tags output with the `dup` command.
---- 11.0.1 Duplicate Tags ----
---- 12.0.1 Duplicate Tags ----
These tags log information about detected duplicate files and provide
a summary of the duplicate finding process. The content hash of the
whole file is used to determine duplicates.
@ -914,7 +931,7 @@ whole file is used to determine duplicates.
<size> - The size of the duplicate file in bytes (uint64). This
size corresponds to the file `<file_path2>`.
---- 11.0.2 Summary Tags ----
---- 12.0.2 Summary Tags ----
These tags provide a final summary of the duplicate finding process.
summary:dup_count:<count>
@ -937,7 +954,7 @@ These tags provide a final summary of the duplicate finding process.
dup - Indicates that at least one duplicate file was found.
12 COMMAND POOL TAGS
13 COMMAND POOL TAGS
====================
This section describes the tags output with the `pool` command.
@ -953,12 +970,12 @@ This section describes the tags output with the `pool` command.
and you won't get this tag.
13 COMMAND REHASH TAGS
14 COMMAND REHASH TAGS
======================
This section describes the tags output with the `rehash` command.
---- 13.0.1 Summary Tags ----
---- 14.0.1 Summary Tags ----
These tags provide a final summary of the rehash scheduling process.
summary:exit:<status>
@ -977,7 +994,7 @@ These tags provide a final summary of the rehash scheduling process.
exit with a fatal error.
14 COMMAND SMART AND PROBE TAGS
15 COMMAND SMART AND PROBE TAGS
===============================
This section describes the tags output by the `smart` and `probe`
@ -996,7 +1013,7 @@ disk.
In case of split parity, the parity name is followed by `/N` to indicate
the split index, where N starts from 0. The `/0` is always omitted.
---- 14.0.1 Attributes ----
---- 15.0.1 Attributes ----
Tags logging specific disk identification and attribute values.
smart:<device_file>:<disk_name>[/<split_index>]
@ -1094,7 +1111,7 @@ Tags logging specific disk identification and attribute values.
<access_number> - Total number of accesses (int64).
---- 14.0.2 Summary ----
---- 15.0.2 Summary ----
Tags logging the overall failure probabilities for the array.
summary:array_failure:<array_afr>:<array_prob>
@ -1107,7 +1124,7 @@ Tags logging the overall failure probabilities for the array.
the next year (double).
15 DEVICE STATUS TAGS
16 DEVICE STATUS TAGS
=====================
These tags log the lifecycle of hardware interactions and external
@ -1143,7 +1160,7 @@ command execution with the `smartctl` command.
process.
16 ERROR TAGS
17 ERROR TAGS
=============
These tags report specific errors that occur on data disks during the