diff --git a/HISTORY b/HISTORY index 542994a1..0bd9e11e 100644 --- a/HISTORY +++ b/HISTORY @@ -48,6 +48,9 @@ SnapRAID HISTORY controllers and USB enclosures. * Added support for specifying case-insensitive SMART attribute names (instead of only numeric IDs) in the `smartignore` configuration option. + * Added support for filesystems where only specific files fail to report a + 64-bit inode number (such as ReFS when 128-bit file IDs cannot be mapped), + allowing SnapRAID to continue processing those individual files without inodes. 14.10 2026/08 ============= diff --git a/TODO b/TODO index 36319276..5c0e630f 100644 --- a/TODO +++ b/TODO @@ -116,14 +116,6 @@ https://sourceforge.net/p/snapraid/discussion/1677233/thread/648955ec/ (Leifi in https://sourceforge.net/p/snapraid/discussion/1677233/thread/5b6ef1b9/ (Miles in check + filter) Partially done. Now you can control it with --test-fmt -* Add ReFS support with 128 bit inode -See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/2be14f63/ -https://github.com/Microsoft/Windows-driver-samples/blob/master/filesys/miniFilter/avscan/filter/utility.h -https://github.com/Microsoft/Windows-driver-samples/blob/master/filesys/miniFilter/avscan/filter/utility.c -https://github.com/Microsoft/Windows-driver-samples/blob/master/filesys/miniFilter/delete/delete.c -Partially done. Until the inodes are less than 64 bit, everything works. -If a 128 bit inode is found, it aborts with "Invalid inode number! Is this ReFS?" - * When a disk is replaced, and SnapRAID detects this by a UUID change, we could clear the scrub information for that disk. https://sourceforge.net/p/snapraid/discussion/1677233/thread/ee87901b/ diff --git a/cmdline/check.c b/cmdline/check.c index 1a80e35e..eb286a06 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -660,7 +660,6 @@ static int file_post(struct snapraid_state* state, int fix, block_off_t i, struc struct snapraid_file* collide_file; struct snapraid_file* file; block_off_t file_pos; - uint64_t inode; disk = handle[j].disk; if (!disk) { @@ -774,10 +773,12 @@ static int file_post(struct snapraid_state* state, int fix, block_off_t i, struc log_tag("status:recovered:%s:%s\n", disk->name, esc_tag(file->sub)); msg_info("recovered %s\n", fmt_term(disk, file->sub)); - inode = handle[j].st.st_ino; - /* search for the corresponding inode */ - collide_file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode)); + uint64_t inode = handle[j].st.st_ino; /* don't know the exact type of st_ino and we cannot pass it by pointer in the search */ + if (inode != INODE_INVALID) + collide_file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode)); + else + collide_file = 0; /* * If the inode is already in the database and it refers to a different file name, @@ -1808,7 +1809,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna log_error(ESOFT, "Error stating hardlink-to '%s' for not regular file.\n", path); log_tag("hardlink_error:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, esc_tag(slink->sub), esc_tag(slink->linkto)); ++soft_error; - } else if (!unsuccessful && st.st_ino != stto.st_ino) { + } else if (!unsuccessful && st.st_ino != INODE_INVALID && stto.st_ino != INODE_INVALID && st.st_ino != stto.st_ino) { unsuccessful = 1; log_error(ESOFT, "Mismatch hardlink '%s' and '%s'. Different inode.\n", path, pathto); diff --git a/cmdline/elem.h b/cmdline/elem.h index 23465ad1..7a1c8018 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -213,14 +213,6 @@ struct snapraid_block { */ #define FILE_IS_UNSYNCED 0x100 -/** - * If the file is without inode. - * It could happen in file-systems where inodes are not persistent, - * or when restoring a full disk with "fix". - * In such cases we have to clear any stored duplicate inode. - * After the scan process completes, no file should have this flag set. - */ -#define FILE_IS_WITHOUT_INODE 0x200 /** * The file is deleted. diff --git a/cmdline/scan.c b/cmdline/scan.c index fa7b0be8..07329502 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -604,7 +604,7 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc st->st_nlink = synced_st.st_nlink; } - if (st->st_ino != synced_st.st_ino) { + if (st->st_ino != INODE_INVALID && synced_st.st_ino != INODE_INVALID && st->st_ino != synced_st.st_ino) { log_tag("%s:%u:%s:%s: Uncached inode change error.\n", es(ESOFT), 0, disk->name, esc_tag(sub)); log_fatal(ESOFT, "DANGER! Detected uncached inode change from %" PRIu64 " to %" PRIu64 " for file '%s'\n", (uint64_t)st->st_ino, (uint64_t)synced_st.st_ino, sub); @@ -633,7 +633,7 @@ static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* f struct snapraid_disk* disk = scan->disk; /* insert the file in the containers */ - if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) + if (file->inode != INODE_INVALID) tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); stamp_lock(disk); @@ -652,9 +652,13 @@ static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* f struct snapraid_disk* disk = scan->disk; /* remove the file from the containers */ - if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) + if (file->inode != INODE_INVALID) tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset); + + stamp_lock(disk); tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset); + tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset); + stamp_unlock(disk); /* * Keep track of the removed file (that won't be added later) @@ -670,10 +674,6 @@ static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* f tommy_list_insert_tail(&disk->dealloclist, &dealloc->nodelist, dealloc); } - stamp_lock(disk); - tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset); - stamp_unlock(disk); - /* * Deallocate the file from the parity. * @@ -763,9 +763,11 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, * Always search with the new inode, in the all new inodes found until now, * with the eventual presence of also the past inodes */ - uint64_t inode = st->st_ino; - - file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode)); + uint64_t inode = st->st_ino; /* don't know the exact type of st_ino and we cannot pass it by pointer in the search */ + if (inode != INODE_INVALID) + file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode)); + else + file = 0; /* identify moved files with past inodes and hardlinks with the new inodes */ if (file) { @@ -901,19 +903,19 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, * otherwise, it will get removed */ + /* here the inode must be present */ + if (file->inode == INODE_INVALID) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Internal inconsistency in inode for file '%s%s' as unexpected missing\n", disk->dir, sub); + os_abort(); + /* LCOV_EXCL_STOP */ + } + /* remove from the inode set */ tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset); - /* - * Clear the inode - * this is not really needed for correct functionality - * because we are going to set FILE_IS_WITHOUT_INODE - * but it's easier for debugging to have invalid inodes set to 0 - */ - file->inode = 0; - - /* mark as missing inode */ - file_flag_set(file, FILE_IS_WITHOUT_INODE); + /* clear the inode */ + file->inode = INODE_INVALID; /* go further to find it by name */ } @@ -931,20 +933,18 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, if (is_file_already_present) { /* if the file is without an inode */ - if (file_flag_has(file, FILE_IS_WITHOUT_INODE)) { + if (file->inode == INODE_INVALID) { /* set it now */ file->inode = st->st_ino; - /* insert in the set */ - tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); - - /* unmark as missing inode */ - file_flag_clear(file, FILE_IS_WITHOUT_INODE); + /* insert in the set if valid */ + if (file->inode != INODE_INVALID) + tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); } else { /* here the inode has to be different, otherwise we would have found it before */ if (file->inode == st->st_ino) { /* LCOV_EXCL_START */ - log_fatal(EINTERNAL, "Internal inconsistency in inode '%" PRIu64 "' for files '%s%s' as unexpected matching\n", file->inode, disk->dir, sub); + log_fatal(EINTERNAL, "Internal inconsistency in inode '%" PRIu64 "' for file '%s%s' as unexpected matching\n", file->inode, disk->dir, sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -987,7 +987,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, } /* if when processing the disk we used the past inodes values */ - if (has_past_inodes) { + if (has_past_inodes && file->inode != INODE_INVALID && st->st_ino != INODE_INVALID) { /* * If persistent inodes are supported, we are sure that the inode number * is now different, because otherwise the file would have been found @@ -1574,7 +1574,7 @@ static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* pa * If the st_ino field is missing, takes care to fill it using the extended lstat() * this can happen only in Windows */ - if (st->st_ino == 0 || st->st_nlink == 0) { + if (st->st_ino == INODE_INVALID || st->st_nlink == 0) { if (lstat_sync(path_next, st, 0) != 0) { /* LCOV_EXCL_START */ log_tag("%s:%u:%s:%s: Stat error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno)); @@ -1736,13 +1736,11 @@ static void* scan_disk(void* arg) node = node->next; /* remove from the inode set */ - tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset); + if (file->inode != INODE_INVALID) + tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset); /* clear the inode */ - file->inode = 0; - - /* mark as missing inode */ - file_flag_set(file, FILE_IS_WITHOUT_INODE); + file->inode = INODE_INVALID; } } diff --git a/cmdline/state.c b/cmdline/state.c index af785e60..6a54318c 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2364,7 +2364,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0); /* insert the file in the file containers */ - tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); + if (file->inode != INODE_INVALID) + tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); diff --git a/cmdline/sync.c b/cmdline/sync.c index 2e7fb3cd..59a379e3 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -191,7 +191,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta if (handle[j].st.st_size != file->size || handle[j].st.st_mtime != file->mtime_sec || STAT_NSEC(&handle[j].st) != file->mtime_nsec - || handle[j].st.st_ino != file->inode + || (handle[j].st.st_ino != INODE_INVALID && file->inode != INODE_INVALID && handle[j].st.st_ino != file->inode) ) { if (handle[j].st.st_size != file->size) { log_tag("error:%" PRIu64 ":%s:%s: Unexpected size change\n", blockcur, disk->name, esc_tag(file->sub)); @@ -564,7 +564,7 @@ static void sync_data_reader(struct snapraid_worker* worker, struct snapraid_tas if (handle->st.st_size != task->file->size || handle->st.st_mtime != task->file->mtime_sec || STAT_NSEC(&handle->st) != task->file->mtime_nsec - || handle->st.st_ino != task->file->inode + || (handle->st.st_ino != INODE_INVALID && task->file->inode != INODE_INVALID && handle->st.st_ino != task->file->inode) ) { log_tag("error:%" PRIu64 ":%s:%s: Unexpected attribute change\n", blockcur, disk->name, esc_tag(task->file->sub)); if (handle->st.st_size != task->file->size) { diff --git a/os/mingw.c b/os/mingw.c index f27a950d..16c59e17 100644 --- a/os/mingw.c +++ b/os/mingw.c @@ -34,9 +34,10 @@ #endif /* File Index */ -#ifndef FILE_INVALID_FILE_ID +#undef FILE_INVALID_FILE_ID #define FILE_INVALID_FILE_ID ((ULONGLONG)-1LL) -#endif +#undef FILE_UNSUPPORTED_FILE_ID +#define FILE_UNSUPPORTED_FILE_ID 0 /** * Direct access to RtlGenRandom(). @@ -464,16 +465,44 @@ static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ * * From Microsoft "Application Compatibility with ReFS" * http://download.microsoft.com/download/C/B/3/CB3561DC-6BF6-443D-B5B9-9676ACDF7F75/Application%20Compatibility%20with%20ReFS.docx - * "64-bit file identifier can be obtained from GetFileInformationByHandle in" - * "the nFileIndexHigh and nFileIndexLow members. This API is an extended version" - * "that includes 128-bit file identifiers. If GetFileInformationByHandle returns" - * "FILE_INVALID_FILE_ID, the identifier may only be described in 128 bit form." + * 64-bit file identifier can be obtained from GetFileInformationByHandle in + * the nFileIndexHigh and nFileIndexLow members. This API is an extended version + * that includes 128-bit file identifiers. If GetFileInformationByHandle returns + * FILE_INVALID_FILE_ID, the identifier may only be described in 128 bit form. + * + * From Microsoft "[MS-FSCC]: File System Control Codes - 11/21/2025 v60.0 - 6 Appendix B: Product Behavior" + * https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-fscc/d4bc551b-7aaf-4b4f-ba0e-3a75e7c528f0 + * + * 64bitFileID Generate Stable Unique + * FAT Yes No No + * EXFAT Yes No No + * FAT32 Yes No No + * UDFS Yes Yes Yes + * NTFS Yes Yes Yes + * ReFS Yes Yes Yes + * + * NTFS computes the 64-bit file ID as follows: the low 48 bits are the index of the + * file's primary record in the master file table (MFT); the remaining 16 bits are + * a sequence number. Therefore, it is possible, though rare, that a different file + * can have the same 64-bit file ID as a file on that volume had in the past. + * + * ReFS maps a subset of the possible 128-bit file ID values to a 64-bit value using + * a reversible algorithm; for values outside of this subset, ReFS sets the 64-bit + * file ID to -1. + * + * From Microsoft "[MS-FSCC]: File System Control Codes - 11/21/2025 v60.0 - 2.1.9 64-bit file ID" + * https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-fscc/2d3333fe-fc98-4a6f-98a2-4bb805aff407?utm_source=chatgpt.com + * + * A 64-bit file ID value uniquely identifies a file within a given volume. This + * identifier is generated and stored by the file system. The identifier SHOULD + * be unique to the volume and stable until the file is deleted. + * For file systems that do not support a 64-bit file ID, this field MUST be + * set to 0, and MUST be ignored. + * For files for which a unique 64-bit file ID cannot be established, this + * field MUST be set to 0xFFFFFFFFFFFFFFFF, and MUST be ignored. */ - if (st->st_ino == (uint64_t)FILE_INVALID_FILE_ID) { - errno = EINVAL; - os_syslog(OS_LVL_INFO, "invalid inode number! Is this ReFS?"); - return -1; - } + if (st->st_ino == FILE_INVALID_FILE_ID || st->st_ino == FILE_UNSUPPORTED_FILE_ID) + st->st_ino = INODE_INVALID; return 0; } @@ -516,11 +545,8 @@ static int windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FIL st->st_sync = 0; /* in ReFS the IDs are 128 bit, and the 64 bit interface may fail */ - if (st->st_ino == (uint64_t)FILE_INVALID_FILE_ID) { - errno = EINVAL; - os_syslog(OS_LVL_INFO, "Invalid inode number! Is this ReFS?"); - return -1; - } + if (st->st_ino == FILE_INVALID_FILE_ID || st->st_ino == FILE_UNSUPPORTED_FILE_ID) + st->st_ino = INODE_INVALID; return 0; } @@ -553,7 +579,7 @@ static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_s st->st_mtimensec = (mtime % 10000000) * 100; /* No inode information available */ - st->st_ino = 0; + st->st_ino = INODE_INVALID; /* No link information available */ st->st_nlink = 0; diff --git a/os/mingw.h b/os/mingw.h index 5282b14e..3d263517 100644 --- a/os/mingw.h +++ b/os/mingw.h @@ -260,7 +260,7 @@ int windows_fstat(int fd, struct windows_stat* st); /** * Like the C lstat(), but with some limitations. * - * The st_ino field may be 0 if it's not possible to read it in a fast way. + * The st_ino field may be INODE_INVALID if it's not possible to read it in a fast way. * Specifically this always happens. * * In case of hardlinks, the size and the attributes of the file can diff --git a/os/os.h b/os/os.h index a1f822b9..8180b3e0 100644 --- a/os/os.h +++ b/os/os.h @@ -195,6 +195,16 @@ int os_validate_exec_input(const char* str); /****************************************************************************/ /* fs */ +/** + * Invalid inode number. + * + * Value 0 is used to represent an invalid or unavailable inode number. + * While 0 is not strictly guaranteed to be invalid by standards, in practice + * it is never used for valid files. And even if a file happens to have inode 0, + * the program will simply process that specific file without using its inode. + */ +#define INODE_INVALID 0 + /** * Physical offset not yet read. */