diff --git a/cmdline/state.c b/cmdline/state.c index 000e7ca3..2ec7c662 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1895,6 +1895,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S unsigned count_hardlink; unsigned count_symlink; unsigned count_dir; + unsigned count_bad; + unsigned count_rehash; int crc_checked; char buffer[PATH_MAX]; int ret; @@ -1906,6 +1908,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S count_hardlink = 0; count_symlink = 0; count_dir = 0; + count_bad = 0; + count_rehash = 0; crc_checked = 0; mapping_max = 0; tommy_array_init(&disk_mapping); @@ -2244,6 +2248,11 @@ static void state_read_content(struct snapraid_state* state, const char* path, S rehash = (flag & 4) != 0; justsynced = (flag & 8) != 0; + if (bad) + ++count_bad; + if (rehash) + ++count_rehash; + if (rehash && state->prevhash == HASH_UNDEFINED) { /* LCOV_EXCL_START */ decoding_error(path, f); @@ -2722,7 +2731,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* LCOV_EXCL_STOP */ } - /* log the info read from the content file */ log_tag("content_data:%s:%s\n", buffer, uuid); @@ -2828,7 +2836,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S state->parity[v_level].total_blocks = v_total_blocks; state->parity[v_level].free_blocks = v_free_blocks; - /* log the info read from the content file */ log_tag("content_parity:%s:%s:%s:%" PRIi64 "\n", lev_config_name(v_level), state->parity[v_level].split_map[0].uuid, @@ -2959,7 +2966,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid); state->parity[v_level].split_map[s].size = v_size; - /* log the info read from the content file */ if (s == 0) pathcpy(parity_name, sizeof(parity_name), lev_config_name(v_level)); else @@ -3045,6 +3051,14 @@ static void state_read_content(struct snapraid_state* state, const char* path, S msg_verbose("%8u hardlinks\n", count_hardlink); msg_verbose("%8u symlinks\n", count_symlink); msg_verbose("%8u empty dirs\n", count_dir); + + log_tag("content_info:file:%u\n", count_file); + log_tag("content_info:hardlink:%u\n", count_hardlink); + log_tag("content_info:symlink:%u\n", count_symlink); + log_tag("content_info:dir_empty:%u\n", count_dir); + log_tag("content_info:block_bad:%u\n", count_bad); + log_tag("content_info:block_rehash:%u\n", count_rehash); + log_tag("content_info:block:%u\n", blockmax); } struct state_write_thread_context { @@ -3058,12 +3072,15 @@ struct state_write_thread_context { time_t info_now; int info_has_rehash; STREAM* f; + int first; /* output */ uint32_t crc; unsigned count_file; unsigned count_hardlink; unsigned count_symlink; unsigned count_dir; + unsigned count_bad; + unsigned count_rehash; }; static void* state_write_thread(void* arg) @@ -3081,6 +3098,8 @@ static void* state_write_thread(void* arg) unsigned count_hardlink; unsigned count_symlink; unsigned count_dir; + unsigned count_bad; + unsigned count_rehash; tommy_node* i; block_off_t idx; block_off_t begin; @@ -3090,6 +3109,8 @@ static void* state_write_thread(void* arg) count_hardlink = 0; count_symlink = 0; count_dir = 0; + count_bad = 0; + count_rehash = 0; /* force version 3 a we want to always store the parity size */ /* write header */ @@ -3182,6 +3203,15 @@ static void* state_write_thread(void* arg) sputb32(map->total_blocks, f); sputb32(map->free_blocks, f); sputbs(map->uuid, f); + if (context->first) { + log_tag("content_allocation:%s:%" PRIi64 ":%" PRIi64 "\n", + map->name, + map->total_blocks * (uint64_t)state->block_size, + map->free_blocks * (uint64_t)state->block_size); + log_tag("content_data:%s:%s\n", + map->name, + map->uuid); + } if (serror(f)) { /* LCOV_EXCL_START */ log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); @@ -3197,11 +3227,29 @@ static void* state_write_thread(void* arg) sputb32(l, f); sputb32(state->parity[l].total_blocks, f); sputb32(state->parity[l].free_blocks, f); + if (context->first) { + log_tag("content_allocation:%s:%" PRIi64 ":%" PRIi64 "\n", + lev_config_name(l), + state->parity[l].total_blocks * (uint64_t)state->block_size, + state->parity[l].free_blocks * (uint64_t)state->block_size); + } sputb32(state->parity[l].split_mac, f); for (s = 0; s < state->parity[l].split_mac; ++s) { + char parity_name[64]; sputbs(state->parity[l].split_map[s].path, f); sputbs(state->parity[l].split_map[s].uuid, f); sputb64(state->parity[l].split_map[s].size, f); + if (s == 0) + pathcpy(parity_name, sizeof(parity_name), lev_config_name(l)); + else + pathprint(parity_name, sizeof(parity_name), "%s/%u", lev_config_name(l), s); + if (context->first) { + log_tag("content_parity:%s:%s:%s:%" PRIi64 "\n", + parity_name, + state->parity[l].split_map[s].uuid, + state->parity[l].split_map[s].path, + state->parity[l].split_map[s].size); + } } if (serror(f)) { /* LCOV_EXCL_START */ @@ -3440,10 +3488,14 @@ static void* state_write_thread(void* arg) if (info) { /* other flags */ flag = 1; /* info is present */ - if (info_get_bad(info)) + if (info_get_bad(info)) { flag |= 2; - if (info_get_rehash(info)) + ++count_bad; + } + if (info_get_rehash(info)) { flag |= 4; + ++count_rehash; + } if (info_get_justsynced(info)) flag |= 8; sputb32(flag, f); @@ -3517,6 +3569,8 @@ static void* state_write_thread(void* arg) context->count_hardlink = count_hardlink; context->count_symlink = count_symlink; context->count_dir = count_dir; + context->count_bad = count_bad; + context->count_rehash = count_rehash; return 0; } @@ -3545,6 +3599,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) unsigned count_hardlink; unsigned count_symlink; unsigned count_dir; + unsigned count_bad; + unsigned count_rehash; /* blocks of all array */ blockmax = parity_allocated_size(state); @@ -3609,14 +3665,17 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) #if HAVE_MT_WRITE /* start all writing threads */ + first = 1; i = tommy_list_head(&state->contentlist); while (i) { struct snapraid_content* content = i->data; struct state_write_thread_context* context; char tmp[PATH_MAX]; + char esc_buffer[ESC_MAX]; STREAM* f; msg_progress("Saving state to %s...\n", content->content); + log_tag("content_write:%s\n", esc_tag(content->content, esc_buffer)); pathprint(tmp, sizeof(tmp), "%s.tmp", content->content); @@ -3649,6 +3708,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) context->info_now = info_now; context->info_has_rehash = info_has_rehash; context->f = f; + context->first = first; + first = 0; thread_create(&context->thread, state_write_thread, context); @@ -3663,6 +3724,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) count_hardlink = 0; count_symlink = 0; count_dir = 0; + count_bad = 0; + count_rehash = 0; i = tommy_list_head(&state->contentlist); while (i) { struct snapraid_content* content = i->data; @@ -3710,6 +3773,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) count_hardlink = context->count_hardlink; count_symlink = context->count_symlink; count_dir = context->count_dir; + cound_bad = context->count_bad; + count_rehash = context->count_rehash; } else { if (crc != context->crc) { /* LCOV_EXCL_START */ @@ -3791,6 +3856,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) context->info_now = info_now; context->info_has_rehash = info_has_rehash; context->f = f; + context->first = 1; retval = state_write_thread(context); @@ -3831,6 +3897,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) count_hardlink = context->count_hardlink; count_symlink = context->count_symlink; count_dir = context->count_dir; + count_bad = context->count_bad; + count_rehash = context->count_rehash; free(context); #endif @@ -3840,6 +3908,14 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) msg_verbose("%8u symlinks\n", count_symlink); msg_verbose("%8u empty dirs\n", count_dir); + log_tag("content_info:file:%u\n", count_file); + log_tag("content_info:hardlink:%u\n", count_hardlink); + log_tag("content_info:symlink:%u\n", count_symlink); + log_tag("content_info:dir_empty:%u\n", count_dir); + log_tag("content_info:block_bad:%u\n", count_bad); + log_tag("content_info:block_rehash:%u\n", count_rehash); + log_tag("content_info:block:%u\n", blockmax); + *out_crc = crc; } @@ -3860,7 +3936,8 @@ void state_read(struct snapraid_state* state) pathcpy(path, sizeof(path), content->content); if (!state->no_conf) { - log_tag("content:%s\n", path); + char esc_buffer[ESC_MAX]; + log_tag("content:%s\n", esc_tag(path, esc_buffer)); log_flush(); } msg_progress("Loading state from %s...\n", path); diff --git a/doc/snapraid_log.1 b/doc/snapraid_log.1 index 133c75c1..c5a23063 100644 --- a/doc/snapraid_log.1 +++ b/doc/snapraid_log.1 @@ -71,8 +71,7 @@ Some tags are output only when the \`\-\-gui\` option is used. They provide extra verbose state and runtime progress only useful for a GUI interface. .SH CONFIGURATION TAGS -These tags report the memory usage and global configuration after -scanning the disk and reading the list of files. +These tags report the memory usage and global configuration. .TP .B conf:file: Configuration file loaded (escaped). @@ -143,12 +142,19 @@ exclude specific files or directories from the SnapRAID array. .B filter:nohidden: If the no\-hidden filter is enabled, meaning hidden files/directories (starting with \'.\') are excluded from the array. +.SH CONTENT TAGS +These tags report the information stored in the content file. .TP .B content: The absolute path to a content file being used (escaped). Content files store the metadata (like file hashes and timestamps) for the array. .TP +.B content_write: +The absolute path to a content file being written (escaped). +You will see multiple writes as the content file is saved in +multiple copies. +.TP .B content_parity:[/]::: The parity files as stored in the content file. .RS 4 @@ -202,6 +208,20 @@ The size of the data in the disk or parity (uint64). The free size in the disk or parity (uint64). .PD .RE +.TP +.B content_info:: +The counters in the content file. +.RS 4 +.PD 0 +.HP 4 +.I +One of file, hardlink, symlink, dir_empty, +block_bad, block_rehash, block. +.HP 4 +.I +The number of elements (uint) +.PD +.RE .SH DIAGNOSTICS TAGS .TP .B version: diff --git a/doc/snapraid_log.d b/doc/snapraid_log.d index 1155607d..2d5f2d48 100644 --- a/doc/snapraid_log.d +++ b/doc/snapraid_log.d @@ -53,8 +53,7 @@ Description Configuration Tags - These tags report the memory usage and global configuration after - scanning the disk and reading the list of files. + These tags report the memory usage and global configuration. =conf:file: Configuration file loaded (escaped). @@ -108,10 +107,18 @@ Configuration Tags If the no-hidden filter is enabled, meaning hidden files/directories (starting with '.') are excluded from the array. +Content Tags + These tags report the information stored in the content file. + =content: The absolute path to a content file being used (escaped). Content files store the metadata (like file hashes and timestamps) for the array. + + =content_write: + The absolute path to a content file being written (escaped). + You will see multiple writes as the content file is saved in + multiple copies. =content_parity:[/]::: The parity files as stored in the content file. @@ -140,6 +147,13 @@ Configuration Tags - The size of the data in the disk or parity (uint64). - The free size in the disk or parity (uint64). + =content_info:: + The counters in the content file. + + - One of file, hardlink, symlink, dir_empty, + block_bad, block_rehash, block. + - The number of elements (uint) + Diagnostics Tags =version: The version of SnapRAID run. diff --git a/doc/snapraid_log.txt b/doc/snapraid_log.txt index cc0c2067..e0522d73 100644 --- a/doc/snapraid_log.txt +++ b/doc/snapraid_log.txt @@ -59,8 +59,7 @@ interface. 2 CONFIGURATION TAGS ==================== -These tags report the memory usage and global configuration after -scanning the disk and reading the list of files. +These tags report the memory usage and global configuration. conf:file: Configuration file loaded (escaped). @@ -114,11 +113,22 @@ scanning the disk and reading the list of files. If the no-hidden filter is enabled, meaning hidden files/directories (starting with '.') are excluded from the array. + +3 CONTENT TAGS +============== + +These tags report the information stored in the content file. + content: The absolute path to a content file being used (escaped). Content files store the metadata (like file hashes and timestamps) for the array. + content_write: + The absolute path to a content file being written (escaped). + You will see multiple writes as the content file is saved in + multiple copies. + content_parity:[/]::: The parity files as stored in the content file. @@ -146,8 +156,15 @@ scanning the disk and reading the list of files. - The size of the data in the disk or parity (uint64). - The free size in the disk or parity (uint64). + content_info:: + The counters in the content file. -3 DIAGNOSTICS TAGS + - One of file, hardlink, symlink, dir_empty, + block_bad, block_rehash, block. + - The number of elements (uint) + + +4 DIAGNOSTICS TAGS ================== version: @@ -186,7 +203,7 @@ scanning the disk and reading the list of files. Size of the internal `snapraid_dir` structure in bytes (uint64). -4 SCAN TAGS +5 SCAN TAGS =========== These tags report detected differences between the filesystem and the @@ -275,7 +292,7 @@ These tags provide a final count of all detected changes. The array needs a `sync` operation. -5 GENERAL PROGRESS AND EXECUTION TAGS +6 GENERAL PROGRESS AND EXECUTION TAGS ===================================== These tags are used to report the runtime processing of the commands @@ -331,7 +348,7 @@ Only produced when `--gui` option is used. - Text message exactly as it's printed on screen. -6 THERMAL TAGS +7 THERMAL TAGS ============== The following tags are produced by the thermal monitoring functions @@ -434,12 +451,12 @@ disk access. are being spun back up to resume the operation. -7 COMMAND STATUS TAGS +8 COMMAND STATUS TAGS ===================== This section describes the tags output with the `status` command. ----- 7.0.1 Summary Tags ---- +---- 8.0.1 Summary Tags ---- These tags provide general information about the array configuration and overall state. @@ -523,7 +540,7 @@ and overall state. unsynced - The array has an interrupted sync. The array needs another `sync` operation. ----- 7.0.2 Per-Disk Tags ---- +---- 8.0.2 Per-Disk Tags ---- These tags provide statistics for individual data disks. is the disk name. @@ -564,14 +581,14 @@ is the disk name. summary:disk_use_percent:: Disk use percentage relative to `disk_block_max`. ----- 7.0.3 File Timestamp Tags ---- +---- 8.0.3 File Timestamp Tags ---- These tags list files with zero sub-second timestamps. zerosubsecond:: Lists files that have a zero sub-second timestamp. Up to 50 are logged per disk. ----- 7.0.4 Block Information Tags ---- +---- 8.0.4 Block Information Tags ---- These tags are generated when the GUI option is enabled (`state->opt.gui`). block_count: @@ -595,7 +612,7 @@ These tags are generated when the GUI option is enabled (`state->opt.gui`). - `used` if the block contains data, otherwise empty. - `unsynced` if the block needs a sync, otherwise empty. ----- 7.0.5 Scrub History Tags ---- +---- 8.0.5 Scrub History Tags ---- These tags provide details on the scrub/sync time distribution. info_count: @@ -630,13 +647,13 @@ These tags provide details on the scrub/sync time distribution. - Count of newly synced (unscrubbed) blocks in that day range. -8 COMMAND SYNC/SCRUB TAGS +9 COMMAND SYNC/SCRUB TAGS ========================= Tags specific for the `sync` and `scrub` commands. Also the error tags are possible. ----- 8.0.1 Summary Tags ---- +---- 9.0.1 Summary Tags ---- These tags provide summary statistics at the end of the command. hash_summary:error_file: @@ -678,13 +695,13 @@ These tags provide summary statistics at the end of the command. or filesystem inspection. -9 COMMAND CHECK/FIX TAGS -======================== +10 COMMAND CHECK/FIX TAGS +========================= Tags specific for the `check` and `fix` commands. Also the error tags are possible. ----- 9.0.1 Fixed/Recovered Tags ---- +---- 10.0.1 Fixed/Recovered Tags ---- These tags log successful repairs during fix operations. fixed:::: @@ -738,7 +755,7 @@ These tags log successful repairs during fix operations. - A descriptive message indicating the nature of the fix (e.g., `Fixed dir error`). ----- 9.0.2 Unrecoverable Error Tags ---- +---- 10.0.2 Unrecoverable Error Tags ---- These tags log errors that could not be recovered. unrecoverable:::: @@ -762,7 +779,7 @@ These tags log errors that could not be recovered. - A descriptive message indicating the nature of the problem (e.g., `Not setting modification time to avoid inode collision`). ----- 9.0.3 File Status Tags ---- +---- 10.0.3 File Status Tags ---- These tags log the final status of files after processing. status:unrecoverable:: @@ -796,7 +813,7 @@ These tags log the final status of files after processing. - The name of the disk. - The file path relative to disk root (escaped). ----- 9.0.4 Summary Tags ---- +---- 10.0.4 Summary Tags ---- These tags provide summary statistics at the end of the command. summary:error: @@ -828,12 +845,12 @@ These tags provide summary statistics at the end of the command. error - Indicates that errors were found (audit-only mode). -10 COMMAND LIST TAGS +11 COMMAND LIST TAGS ==================== This section describes the tags output with the `list` command. ----- 10.0.1 List Tags ---- +---- 11.0.1 List Tags ---- These tags log details about individual files and links processed during the listing operation. @@ -862,7 +879,7 @@ the listing operation. (escaped). - The target path of the link (escaped). ----- 10.0.2 Summary Tags ---- +---- 11.0.2 Summary Tags ---- These tags log final statistics about the entire listing operation. summary:file_count: @@ -887,12 +904,12 @@ These tags log final statistics about the entire listing operation. and you won't get this tag. -11 COMMAND DUP TAGS +12 COMMAND DUP TAGS =================== This section describes the tags output with the `dup` command. ----- 11.0.1 Duplicate Tags ---- +---- 12.0.1 Duplicate Tags ---- These tags log information about detected duplicate files and provide a summary of the duplicate finding process. The content hash of the whole file is used to determine duplicates. @@ -914,7 +931,7 @@ whole file is used to determine duplicates. - The size of the duplicate file in bytes (uint64). This size corresponds to the file ``. ----- 11.0.2 Summary Tags ---- +---- 12.0.2 Summary Tags ---- These tags provide a final summary of the duplicate finding process. summary:dup_count: @@ -937,7 +954,7 @@ These tags provide a final summary of the duplicate finding process. dup - Indicates that at least one duplicate file was found. -12 COMMAND POOL TAGS +13 COMMAND POOL TAGS ==================== This section describes the tags output with the `pool` command. @@ -953,12 +970,12 @@ This section describes the tags output with the `pool` command. and you won't get this tag. -13 COMMAND REHASH TAGS +14 COMMAND REHASH TAGS ====================== This section describes the tags output with the `rehash` command. ----- 13.0.1 Summary Tags ---- +---- 14.0.1 Summary Tags ---- These tags provide a final summary of the rehash scheduling process. summary:exit: @@ -977,7 +994,7 @@ These tags provide a final summary of the rehash scheduling process. exit with a fatal error. -14 COMMAND SMART AND PROBE TAGS +15 COMMAND SMART AND PROBE TAGS =============================== This section describes the tags output by the `smart` and `probe` @@ -996,7 +1013,7 @@ disk. In case of split parity, the parity name is followed by `/N` to indicate the split index, where N starts from 0. The `/0` is always omitted. ----- 14.0.1 Attributes ---- +---- 15.0.1 Attributes ---- Tags logging specific disk identification and attribute values. smart::[/] @@ -1094,7 +1111,7 @@ Tags logging specific disk identification and attribute values. - Total number of accesses (int64). ----- 14.0.2 Summary ---- +---- 15.0.2 Summary ---- Tags logging the overall failure probabilities for the array. summary:array_failure:: @@ -1107,7 +1124,7 @@ Tags logging the overall failure probabilities for the array. the next year (double). -15 DEVICE STATUS TAGS +16 DEVICE STATUS TAGS ===================== These tags log the lifecycle of hardware interactions and external @@ -1143,7 +1160,7 @@ command execution with the `smartctl` command. process. -16 ERROR TAGS +17 ERROR TAGS ============= These tags report specific errors that occur on data disks during the