mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 19:01:57 -04:00
2bac137218
The new content file format now includes metadata for previous versions of files. This allows the fix command to access historical file data from snapshots as needed, ensuring recovery even if the live filesystem has changed or the file was deleted during a sync.
2177 lines
65 KiB
C
2177 lines
65 KiB
C
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
// Copyright (C) 2011 Andrea Mazzoleni
|
|
|
|
#include "portable.h"
|
|
|
|
#include "support.h"
|
|
#include "elem.h"
|
|
#include "state.h"
|
|
#include "parity.h"
|
|
|
|
static const char* es(int err)
|
|
{
|
|
if (is_hw(err))
|
|
return "error_io";
|
|
else
|
|
return "error";
|
|
}
|
|
|
|
struct snapraid_scan {
|
|
struct snapraid_state* state; /**< State used. */
|
|
struct snapraid_disk* disk; /**< Disk used. */
|
|
thread_id_t thread; /**< Thread used for scanning the disk */
|
|
|
|
int is_diff; /**< If it's a diff command or a scanning */
|
|
int need_write; /**< If a state write is required */
|
|
|
|
/**
|
|
* Counters of changes.
|
|
*/
|
|
unsigned count_equal; /**< Files equal. */
|
|
unsigned count_move; /**< Files with a different name, but equal inode, size and timestamp in the same disk. */
|
|
unsigned count_restore; /**< Files with equal name, size and timestamp, but different inode. */
|
|
unsigned count_change; /**< Files with same name, but different size and/or timestamp. */
|
|
unsigned count_copy; /**< Files new, with same name size and timestamp of a file in a different disk. */
|
|
unsigned count_relocate; /**< Like copy, but with the original disappeared. */
|
|
unsigned count_insert; /**< Files new. */
|
|
unsigned count_remove; /**< Files removed. */
|
|
|
|
tommy_list file_insert_list; /**< Files to insert. */
|
|
tommy_list link_insert_list; /**< Links to insert. */
|
|
tommy_list dir_insert_list; /**< Dirs to insert. */
|
|
tommy_list local_filter_list; /**< Filter list specific for the disk. */
|
|
|
|
/* nodes for data structures */
|
|
tommy_node node;
|
|
};
|
|
|
|
static struct snapraid_scan* scan_alloc(struct snapraid_state* state, struct snapraid_disk* disk, int is_diff)
|
|
{
|
|
struct snapraid_scan* scan;
|
|
|
|
scan = malloc_nofail(sizeof(struct snapraid_scan));
|
|
scan->state = state;
|
|
scan->disk = disk;
|
|
scan->count_equal = 0;
|
|
scan->count_move = 0;
|
|
scan->count_copy = 0;
|
|
scan->count_relocate = 0;
|
|
scan->count_restore = 0;
|
|
scan->count_change = 0;
|
|
scan->count_remove = 0;
|
|
scan->count_insert = 0;
|
|
tommy_list_init(&scan->file_insert_list);
|
|
tommy_list_init(&scan->link_insert_list);
|
|
tommy_list_init(&scan->dir_insert_list);
|
|
tommy_list_init(&scan->local_filter_list);
|
|
scan->is_diff = is_diff;
|
|
scan->need_write = 0;
|
|
|
|
#if HAVE_THREAD
|
|
thread_mutex_init(&disk->stamp_mutex);
|
|
#endif
|
|
|
|
return scan;
|
|
}
|
|
|
|
static void scan_free(struct snapraid_scan* scan)
|
|
{
|
|
#if HAVE_THREAD
|
|
thread_mutex_destroy(&scan->disk->stamp_mutex);
|
|
#endif
|
|
tommy_list_foreach(&scan->local_filter_list, filter_free);
|
|
free(scan);
|
|
}
|
|
|
|
static void stamp_lock(struct snapraid_disk* disk)
|
|
{
|
|
#if HAVE_THREAD
|
|
thread_mutex_lock(&disk->stamp_mutex);
|
|
#else
|
|
(void)disk;
|
|
#endif
|
|
}
|
|
|
|
static void stamp_unlock(struct snapraid_disk* disk)
|
|
{
|
|
#if HAVE_THREAD
|
|
thread_mutex_unlock(&disk->stamp_mutex);
|
|
#else
|
|
(void)disk;
|
|
#endif
|
|
}
|
|
|
|
/**
|
|
* Remove the specified link from the data set.
|
|
*/
|
|
static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* slink)
|
|
{
|
|
struct snapraid_disk* disk = scan->disk;
|
|
|
|
/* state changed */
|
|
scan->need_write = 1;
|
|
|
|
/* remove the file from the link containers */
|
|
tommy_hashdyn_remove_existing(&disk->linkset, &slink->nodeset);
|
|
tommy_list_remove_existing(&disk->linklist, &slink->nodelist);
|
|
|
|
/* deallocate */
|
|
link_free(slink);
|
|
}
|
|
|
|
/**
|
|
* Insert the specified link in the data set.
|
|
*/
|
|
static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* slink)
|
|
{
|
|
struct snapraid_disk* disk = scan->disk;
|
|
|
|
/* state changed */
|
|
scan->need_write = 1;
|
|
|
|
/* insert the link in the link containers */
|
|
tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
|
|
tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
|
|
}
|
|
|
|
/**
|
|
* Process a symbolic link.
|
|
*/
|
|
static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub, const char* linkto, unsigned link_flag)
|
|
{
|
|
struct snapraid_state* state = scan->state;
|
|
struct snapraid_disk* disk = scan->disk;
|
|
struct snapraid_link* slink;
|
|
/* check if the link already exists */
|
|
slink = tommy_hashdyn_search(&disk->linkset, link_name_compare_to_arg, sub, link_name_hash(sub));
|
|
if (slink) {
|
|
/* check if multiple files have the same name */
|
|
if (link_flag_has(slink, FILE_IS_PRESENT)) {
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inconsistency for link '%s%s'\n", disk->dir, sub);
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* mark as present */
|
|
link_flag_set(slink, FILE_IS_PRESENT);
|
|
|
|
/* check if the link is not changed and it's of the same kind */
|
|
if (strcmp(slink->linkto, linkto) == 0 && link_flag == link_flag_get(slink, FILE_IS_LINK_MASK)) {
|
|
/* it's equal */
|
|
++scan->count_equal;
|
|
|
|
if (state->opt.gui_verbose) {
|
|
log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(slink->sub));
|
|
}
|
|
} else {
|
|
/* it's an update */
|
|
|
|
/* we have to save the linkto/type */
|
|
scan->need_write = 1;
|
|
|
|
++scan->count_change;
|
|
|
|
log_tag("scan:update:%s:%s\n", disk->name, esc_tag(slink->sub));
|
|
if (is_diff) {
|
|
msg_info("update %s\n", fmt_term(disk, slink->sub));
|
|
}
|
|
|
|
/* update it */
|
|
free(slink->linkto);
|
|
slink->linkto = strdup_nofail(linkto);
|
|
link_flag_let(slink, link_flag, FILE_IS_LINK_MASK);
|
|
}
|
|
|
|
/* nothing more to do */
|
|
return;
|
|
} else {
|
|
/* create the new link */
|
|
++scan->count_insert;
|
|
|
|
log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub));
|
|
if (is_diff) {
|
|
msg_info("add %s\n", fmt_term(disk, sub));
|
|
}
|
|
|
|
/* and continue to insert it */
|
|
}
|
|
|
|
/* insert it */
|
|
slink = link_alloc(sub, linkto, link_flag);
|
|
|
|
/* mark it as present */
|
|
link_flag_set(slink, FILE_IS_PRESENT);
|
|
|
|
/* insert it in the delayed insert list */
|
|
tommy_list_insert_tail(&scan->link_insert_list, &slink->nodelist, slink);
|
|
}
|
|
|
|
/**
|
|
* Insert the specified file in the parity.
|
|
*/
|
|
static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
|
|
{
|
|
struct snapraid_state* state = scan->state;
|
|
struct snapraid_disk* disk = scan->disk;
|
|
block_off_t i;
|
|
block_off_t parity_pos;
|
|
|
|
/* state changed */
|
|
scan->need_write = 1;
|
|
|
|
/* allocate the blocks of the file */
|
|
parity_pos = disk->first_free_block;
|
|
for (i = 0; i < file->blockmax; ++i) {
|
|
struct snapraid_block* block;
|
|
struct snapraid_block* over_block;
|
|
snapraid_info info;
|
|
|
|
/* increment the position until the first really free block */
|
|
while (block_has_file(fs_par2block_find(disk, parity_pos)))
|
|
++parity_pos;
|
|
|
|
/* get block we are going to overwrite, if any */
|
|
over_block = fs_par2block_find(disk, parity_pos);
|
|
|
|
/* deallocate it */
|
|
if (over_block != BLOCK_NULL)
|
|
fs_deallocate(disk, parity_pos);
|
|
|
|
/* get block specific info */
|
|
info = info_get(&state->infoarr, parity_pos);
|
|
|
|
/* get the new block we are going to write */
|
|
block = fs_file2block_get(file, i);
|
|
|
|
/* if the file block already has an updated hash without rehash */
|
|
if (block_has_updated_hash(block) && !info_get_rehash(info)) {
|
|
/* the only possible case is for REP blocks */
|
|
assert(block_state_get(block) == BLOCK_STATE_REP);
|
|
|
|
/* convert to a REP block */
|
|
block_state_set(block, BLOCK_STATE_REP);
|
|
|
|
/* and keep the hash as it's */
|
|
} else {
|
|
unsigned over_state;
|
|
|
|
/* convert to a CHG block */
|
|
block_state_set(block, BLOCK_STATE_CHG);
|
|
|
|
/* state of the block we are going to overwrite */
|
|
over_state = block_state_get(over_block);
|
|
|
|
/* if the block is an empty one */
|
|
if (over_state == BLOCK_STATE_EMPTY) {
|
|
/*
|
|
* The block was empty and filled with zeros
|
|
* set the hash to the special ZERO value
|
|
*/
|
|
hash_zero_set(block->hash);
|
|
} else {
|
|
/* otherwise it's a DELETED one */
|
|
assert(over_state == BLOCK_STATE_DELETED);
|
|
|
|
/*
|
|
* In this cases we don't know if the old state is still the one
|
|
* stored inside the parity, because after an aborted sync, the parity
|
|
* may be or may be not have been updated with the data that it's now
|
|
* deleted.
|
|
*
|
|
* We anyway keep the hash as the new sync will not assume it correct,
|
|
* but the check/fix may be using it.
|
|
*
|
|
* For example:
|
|
* - One file is deleted
|
|
* - Sync aborted after updating the parity to the new state,
|
|
* but without saving the content file representing this new state.
|
|
* - Another file is added again (exactly here)
|
|
* with the hash of DELETED block not representing the real parity state
|
|
*/
|
|
|
|
/* copy the past hash of the block */
|
|
memcpy(block->hash, over_block->hash, BLOCK_HASH_SIZE);
|
|
}
|
|
}
|
|
|
|
/* store in the disk map, after invalidating all the other blocks */
|
|
fs_allocate(disk, parity_pos, file, i);
|
|
|
|
/* set the new free position */
|
|
disk->first_free_block = parity_pos + 1;
|
|
}
|
|
|
|
/* insert in the list of contained files */
|
|
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
|
|
}
|
|
|
|
/**
|
|
* Delete the specified file from the parity.
|
|
*
|
|
* Note that the parity remains allocated, but the blocks and the file are marked as DELETED.
|
|
*
|
|
* The file pointer remain in memory and it's inserted in the deleted set.
|
|
*
|
|
* It should not be deallocated, as the parity still references it, and other threads
|
|
* in the scanning process may reference it when searching for a copy in the "stampset".
|
|
*/
|
|
static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
|
|
{
|
|
struct snapraid_disk* disk = scan->disk;
|
|
block_off_t i;
|
|
|
|
/* remove from the list of contained files */
|
|
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
|
|
|
|
/* state changed */
|
|
scan->need_write = 1;
|
|
|
|
/*
|
|
* Here we are supposed to adjust the ::first_free_block position
|
|
* with the parity position we are deleting
|
|
* but we also know that we do only delayed insert, after all the deletion,
|
|
* so at this point ::first_free_block is always at 0, and we don't need to update it
|
|
*/
|
|
if (disk->first_free_block != 0) {
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inconsistency for first free position at '%u' deallocating file '%s'\n", disk->first_free_block, file->sub);
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* free all the blocks of the file */
|
|
for (i = 0; i < file->blockmax; ++i) {
|
|
struct snapraid_block* block = fs_file2block_get(file, i);
|
|
unsigned block_state;
|
|
|
|
/*
|
|
* In case we scan after an aborted sync,
|
|
* we could get also intermediate states
|
|
*/
|
|
block_state = block_state_get(block);
|
|
switch (block_state) {
|
|
case BLOCK_STATE_BLK :
|
|
/* we keep the hash making it an "old" hash, because the parity is still containing data for it */
|
|
break;
|
|
case BLOCK_STATE_CHG :
|
|
/*
|
|
* In this cases we don't know if the old state is still the one
|
|
* stored inside the parity, because after an aborted sync, the parity
|
|
* may be or may be not have been updated with the data that it's now
|
|
* deleted.
|
|
*
|
|
* We anyway keep the hash as the new sync will not assume it correct,
|
|
* but the check/fix may be using it.
|
|
*
|
|
* For example:
|
|
* - One file is added
|
|
* - Sync aborted after updating the parity to the new state,
|
|
* but without saving the content file representing this new state.
|
|
* - File is now deleted after the aborted sync
|
|
* - Sync again, deleting the blocks (exactly here)
|
|
* with the hash of CHG block not representing the real parity state
|
|
*/
|
|
break;
|
|
case BLOCK_STATE_REP :
|
|
/* we just don't know the old hash, and then we set it to invalid */
|
|
hash_invalid_set(block->hash);
|
|
break;
|
|
default :
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inconsistency in file '%s' deallocating block '%u:%u' state %u\n", file->sub, i, file->blockmax, block_state);
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* set the block as deleted */
|
|
block_state_set(block, BLOCK_STATE_DELETED);
|
|
}
|
|
|
|
/* mark the file as deleted */
|
|
file_flag_set(file, FILE_IS_DELETED);
|
|
|
|
/* insert it in the list of deleted blocks */
|
|
tommy_list_insert_tail(&disk->deletedlist, &file->nodelist, file);
|
|
}
|
|
|
|
static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
|
|
{
|
|
struct snapraid_state* state = scan->state;
|
|
struct snapraid_disk* disk = scan->disk;
|
|
|
|
/* if we sort for physical offsets we have to read them for new files */
|
|
if (state->opt.force_order == SORT_PHYSICAL
|
|
&& file->physical == FILEPHY_UNREAD_OFFSET
|
|
) {
|
|
char path_next[PATH_MAX];
|
|
|
|
pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, file->sub);
|
|
|
|
if (filephy(path_next, file->size, &file->physical) != 0) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: File physycal offset error. %s.\n", es(errno), 0, disk->name, esc_tag(file->sub), strerror(errno));
|
|
log_fatal(errno, "Error in getting the physical offset of file '%s'. %s.\n", path_next, strerror(errno));
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
}
|
|
|
|
/* insert in the delayed list */
|
|
tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
|
|
}
|
|
|
|
/**
|
|
* Check if a file is completely formed of blocks with invalid parity,
|
|
* and no rehash is tagged, and if it has at least one block.
|
|
*/
|
|
static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
|
|
{
|
|
block_off_t i;
|
|
|
|
/* with no block, it never has an invalid parity */
|
|
if (file->blockmax == 0)
|
|
return 0;
|
|
|
|
/* check all blocks */
|
|
for (i = 0; i < file->blockmax; ++i) {
|
|
snapraid_info info;
|
|
struct snapraid_block* block = fs_file2block_get(file, i);
|
|
block_off_t parity_pos;
|
|
|
|
/* exclude blocks with parity */
|
|
if (!block_has_invalid_parity(block))
|
|
return 0;
|
|
|
|
/*
|
|
* Get the parity position.
|
|
*
|
|
* Note that here we expect to always have mapped
|
|
* parity, because kept files always have it.
|
|
*
|
|
* Anyway, checking for POS_NULL doesn't hurt.
|
|
*/
|
|
parity_pos = fs_file2par_find(disk, file, i);
|
|
|
|
/* if it's not mapped, it cannot have rehash */
|
|
if (parity_pos != POS_NULL) {
|
|
/* get block specific info */
|
|
info = info_get(&state->infoarr, parity_pos);
|
|
|
|
/* if rehash fails */
|
|
if (info_get_rehash(info))
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
/**
|
|
* Check if a file is completely formed of blocks with an updated hash,
|
|
* and no rehash is tagged, and if it has at least one block.
|
|
*/
|
|
static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
|
|
{
|
|
block_off_t i;
|
|
|
|
/* with no block, it never has a hash */
|
|
if (file->blockmax == 0)
|
|
return 0;
|
|
|
|
/* check all blocks */
|
|
for (i = 0; i < file->blockmax; ++i) {
|
|
snapraid_info info;
|
|
struct snapraid_block* block = fs_file2block_get(file, i);
|
|
block_off_t parity_pos;
|
|
|
|
/* exclude blocks without hash */
|
|
if (!block_has_updated_hash(block))
|
|
return 0;
|
|
|
|
/*
|
|
* Get the parity position.
|
|
*
|
|
* Note that it's possible to have files
|
|
* not mapped into the parity, even if they
|
|
* have a valid hash.
|
|
*
|
|
* This happens for example, for 'copied' files
|
|
* that have REP blocks, but not yet mapped.
|
|
*
|
|
* If there are multiple copies, it's also possible
|
|
* that such files are used as 'source' to copy
|
|
* hashes, and then to get them inside this function.
|
|
*/
|
|
parity_pos = fs_file2par_find(disk, file, i);
|
|
|
|
/* if it's not mapped, it cannot have rehash */
|
|
if (parity_pos != POS_NULL) {
|
|
/* get block specific info */
|
|
info = info_get(&state->infoarr, parity_pos);
|
|
|
|
/* exclude blocks needing a rehash */
|
|
if (info_get_rehash(info))
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
/**
|
|
* Refresh the file info.
|
|
*
|
|
* This is needed by Windows as the normal way to list directories may report not
|
|
* updated info. Only the GetFileInformationByHandle() func, called file-by-file,
|
|
* really ensures to return synced info.
|
|
*
|
|
* If this happens, we read also the physical offset, to avoid to read it later.
|
|
*/
|
|
static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struct stat* st, uint64_t* physical)
|
|
{
|
|
#if HAVE_LSTAT_SYNC
|
|
struct snapraid_state* state = scan->state;
|
|
struct snapraid_disk* disk = scan->disk;
|
|
|
|
/* if the st_sync is not set, ensure to get synced info */
|
|
if (st->st_sync == 0) {
|
|
char path_next[PATH_MAX];
|
|
struct stat synced_st;
|
|
|
|
pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, sub);
|
|
|
|
/* if we sort for physical offsets we have to read them for new files */
|
|
if (state->opt.force_order == SORT_PHYSICAL
|
|
&& *physical == FILEPHY_UNREAD_OFFSET
|
|
) {
|
|
/* do nothing, leave the pointer to read the physical offset */
|
|
} else {
|
|
physical = 0; /* set the pointer to 0 to read nothing */
|
|
}
|
|
|
|
if (lstat_sync(path_next, &synced_st, physical) != 0) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: Stat error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno));
|
|
log_fatal(errno, "Error in stat file '%s'. %s.\n", path_next, strerror(errno));
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
if (st->st_mtime != synced_st.st_mtime
|
|
|| st->st_mtimensec != synced_st.st_mtimensec
|
|
) {
|
|
#ifndef _WIN32
|
|
/*
|
|
* In Windows having different metadata is expected with open files
|
|
* because the metadata in the directory is updated only when the file
|
|
* is closed.
|
|
*
|
|
* The same happens for hardlinks that duplicate metadata.
|
|
* The link metadata is updated only when the link is opened.
|
|
* This extends also to st_size and st_nlink.
|
|
*
|
|
* See also:
|
|
* Why is the file size reported incorrectly for files that are still being written to?
|
|
* http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx
|
|
*/
|
|
log_tag("%s:%u:%s:%s: Uncached time change error.\n", es(ESOFT), 0, disk->name, esc_tag(sub));
|
|
log_error(ESOFT, "WARNING! Detected uncached time change from %" PRIu64 ".%09u to %" PRIu64 ".%09u for file '%s'\n",
|
|
(uint64_t)st->st_mtime, (uint32_t)st->st_mtimensec, (uint64_t)synced_st.st_mtime, (uint32_t)synced_st.st_mtimensec, sub);
|
|
log_error(ESOFT, "It's better if you run SnapRAID without other processes running.\n");
|
|
#endif
|
|
st->st_mtime = synced_st.st_mtime;
|
|
st->st_mtimensec = synced_st.st_mtimensec;
|
|
}
|
|
|
|
if (st->st_size != synced_st.st_size) {
|
|
#ifndef _WIN32
|
|
log_tag("%s:%u:%s:%s: Uncached size change error.\n", es(ESOFT), 0, disk->name, esc_tag(sub));
|
|
log_error(ESOFT, "WARNING! Detected uncached size change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
|
|
(uint64_t)st->st_size, (uint64_t)synced_st.st_size, sub);
|
|
log_error(ESOFT, "It's better if you run SnapRAID without other processes running.\n");
|
|
#endif
|
|
st->st_size = synced_st.st_size;
|
|
}
|
|
|
|
if (st->st_nlink != synced_st.st_nlink) {
|
|
#ifndef _WIN32
|
|
log_tag("%s:%u:%s:%s: Uncached nlink change error.\n", es(ESOFT), 0, disk->name, esc_tag(sub));
|
|
log_error(ESOFT, "WARNING! Detected uncached nlink change from %u to %u for file '%s'\n",
|
|
(uint32_t)st->st_nlink, (uint32_t)synced_st.st_nlink, sub);
|
|
log_error(ESOFT, "It's better if you run SnapRAID without other processes running.\n");
|
|
#endif
|
|
st->st_nlink = synced_st.st_nlink;
|
|
}
|
|
|
|
if (st->st_ino != synced_st.st_ino) {
|
|
log_tag("%s:%u:%s:%s: Uncached inode change error.\n", es(ESOFT), 0, disk->name, esc_tag(sub));
|
|
log_fatal(ESOFT, "DANGER! Detected uncached inode change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
|
|
(uint64_t)st->st_ino, (uint64_t)synced_st.st_ino, sub);
|
|
log_fatal(ESOFT, "It's better if you run SnapRAID without other processes running.\n");
|
|
/*
|
|
* At this point, it's too late to change inode
|
|
* and having inconsistent inodes may result to internal failures
|
|
* so, it's better to abort
|
|
*/
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
#else
|
|
(void)scan;
|
|
(void)sub;
|
|
(void)st;
|
|
(void)physical;
|
|
#endif
|
|
}
|
|
|
|
/**
|
|
* Insert the file in the data set.
|
|
*/
|
|
static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
|
|
{
|
|
struct snapraid_disk* disk = scan->disk;
|
|
|
|
/* insert the file in the containers */
|
|
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
|
|
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
|
|
|
|
stamp_lock(disk);
|
|
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
|
|
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
|
|
stamp_unlock(disk);
|
|
|
|
/* delayed allocation of the parity */
|
|
scan_file_delayed_allocate(scan, file);
|
|
}
|
|
|
|
/**
|
|
* Remove the file from the data set.
|
|
*
|
|
* File is then deleted.
|
|
*/
|
|
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file, int keep_track)
|
|
{
|
|
struct snapraid_disk* disk = scan->disk;
|
|
|
|
/* remove the file from the containers */
|
|
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
|
|
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
|
|
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
|
|
|
|
/*
|
|
* Keep track of the removed file (that won't be added later)
|
|
*
|
|
* This allows to record from where the parity was computed.
|
|
*/
|
|
if (keep_track) {
|
|
struct snapraid_dealloc* dealloc = dealloc_alloc(scan->state->block_size, file->sub, file->size, file->mtime_sec, file->mtime_nsec);
|
|
|
|
dealloc_import(dealloc, file);
|
|
|
|
/* insert the dealloc in the dealloc containers */
|
|
tommy_list_insert_tail(&disk->dealloclist, &dealloc->nodelist, dealloc);
|
|
}
|
|
|
|
stamp_lock(disk);
|
|
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
|
|
stamp_unlock(disk);
|
|
|
|
/* deallocate the file from the parity */
|
|
scan_file_deallocate(scan, file);
|
|
}
|
|
|
|
/**
|
|
* Keep the file as it's (or with only a name/inode modification).
|
|
*
|
|
* If the file is kept, nothing has to be done.
|
|
*
|
|
* But if a file contains only blocks with invalid parity, it's reallocated to ensure
|
|
* to always minimize the space used in the parity.
|
|
*
|
|
* This could happen after a failed sync, when some other files are deleted,
|
|
* and then new ones can be moved backward to fill the hole created.
|
|
*/
|
|
static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* file)
|
|
{
|
|
struct snapraid_disk* disk = scan->disk;
|
|
|
|
/* if the file is full invalid, schedule a reinsert at later stage */
|
|
if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) {
|
|
struct snapraid_file* copy = file_dup(file);
|
|
|
|
/* remove the file */
|
|
scan_file_remove(scan, file, 0);
|
|
|
|
/* reinsert the copy in the delayed list */
|
|
scan_file_insert(scan, copy);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Process a file.
|
|
*/
|
|
static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, struct stat* st, uint64_t physical)
|
|
{
|
|
struct snapraid_state* state = scan->state;
|
|
struct snapraid_disk* disk = scan->disk;
|
|
struct snapraid_file* file;
|
|
tommy_node* i;
|
|
int is_original_file_size_different_than_zero;
|
|
int is_file_already_present;
|
|
data_off_t file_already_present_size;
|
|
int64_t file_already_present_mtime_sec;
|
|
int file_already_present_mtime_nsec;
|
|
int is_file_reported;
|
|
/*
|
|
* If the disk has persistent inodes and UUID, try a search on the past inodes,
|
|
* to detect moved files.
|
|
*
|
|
* For persistent inodes we mean inodes that keep their values when the file-system
|
|
* is unmounted and remounted. This don't always happen.
|
|
*
|
|
* Cases found are:
|
|
* - Linux FUSE with exFAT driver from https://code.google.com/p/exfat/.
|
|
* Inodes are reassigned at every mount restarting from 1 and incrementing.
|
|
* As worse, the exFAT support in FUSE doesn't use sub-second precision in timestamps
|
|
* making inode collision more easy (exFAT by design supports 10ms precision).
|
|
* - Linux VFAT kernel (3.2) driver. Inodes are fully reassigned at every mount.
|
|
*
|
|
* In such cases, to avoid possible random collisions, it's better to disable the moved
|
|
* file recognition.
|
|
*
|
|
* For persistent UUID we mean that it has the same UUID as before.
|
|
* Otherwise, if the UUID is changed, likely it's a new recreated file-system,
|
|
* and then the inode have no meaning.
|
|
*
|
|
* Note that to disable the search by past inode, we do this implicitly
|
|
* removing all the past inode before searching for files.
|
|
* This ensures that no file is found with a past inode, but at the same time,
|
|
* it allows to find new files with the same inode, to identify them as hardlinks.
|
|
*/
|
|
int has_past_inodes = !disk->has_volatile_inodes && !disk->has_different_uuid && !disk->has_unsupported_uuid;
|
|
|
|
/*
|
|
* Always search with the new inode, in the all new inodes found until now,
|
|
* with the eventual presence of also the past inodes
|
|
*/
|
|
uint64_t inode = st->st_ino;
|
|
|
|
file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
|
|
|
|
/* identify moved files with past inodes and hardlinks with the new inodes */
|
|
if (file) {
|
|
/* check if the file is not changed */
|
|
if (file->size == st->st_size
|
|
&& file->mtime_sec == st->st_mtime
|
|
&& (file->mtime_nsec == STAT_NSEC(st)
|
|
/*
|
|
* Always accept the stored value if it's STAT_NSEC_INVALID
|
|
* it happens when upgrading from an old version of SnapRAID
|
|
* not yet supporting the nanosecond field
|
|
*/
|
|
|| file->mtime_nsec == STAT_NSEC_INVALID
|
|
)
|
|
) {
|
|
/* check if multiple files have the same inode */
|
|
if (file_flag_has(file, FILE_IS_PRESENT)) {
|
|
/* if has_volatile_hardlinks is true, the nlink value is not reliable */
|
|
if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub);
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* it's a hardlink */
|
|
scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
|
|
return;
|
|
}
|
|
|
|
/* mark as present */
|
|
file_flag_set(file, FILE_IS_PRESENT);
|
|
|
|
/*
|
|
* Update the nanoseconds mtime only if different
|
|
* to avoid unneeded updates
|
|
*/
|
|
if (file->mtime_nsec == STAT_NSEC_INVALID
|
|
&& STAT_NSEC(st) != file->mtime_nsec
|
|
) {
|
|
file->mtime_nsec = STAT_NSEC(st);
|
|
|
|
/* we have to save the new mtime */
|
|
scan->need_write = 1;
|
|
}
|
|
|
|
if (strcmp(file->sub, sub) != 0) {
|
|
/* if the path is different, it means a moved file with the same inode */
|
|
++scan->count_move;
|
|
|
|
log_tag("scan:move:%s:%s:%s\n", disk->name, esc_tag(file->sub), esc_tag(sub));
|
|
if (is_diff) {
|
|
msg_info("move %s -> %s\n", fmt_term(disk, file->sub), fmt_term(disk, sub));
|
|
}
|
|
|
|
/* remove from the name set */
|
|
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
|
|
|
|
/* save the new name */
|
|
file_rename(file, sub);
|
|
|
|
/* reinsert in the name set */
|
|
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
|
|
|
|
/* we have to save the new name */
|
|
scan->need_write = 1;
|
|
} else {
|
|
/* otherwise it's equal */
|
|
++scan->count_equal;
|
|
|
|
if (state->opt.gui_verbose) {
|
|
log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub));
|
|
}
|
|
}
|
|
|
|
/* mark the file as kept */
|
|
scan_file_keep(scan, file);
|
|
|
|
/* nothing more to do */
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Here the file matches the inode, but not the other info
|
|
*
|
|
* It could be a modified file with the same name,
|
|
* or a restored/copied file that get assigned a previously used inode,
|
|
* or a file-system with not persistent inodes.
|
|
*
|
|
* In NTFS it could be also a hardlink, because in NTFS
|
|
* hardlink don't share the same directory information,
|
|
* like attribute and time.
|
|
*
|
|
* For example:
|
|
* C:> echo A > A
|
|
* C:> mklink /H B A
|
|
* ...wait one minute
|
|
* C:> echo AAAAAAAAAAAAAA > A
|
|
* C:> dir
|
|
* ...both time and size of A and B don't match!
|
|
*/
|
|
if (file_flag_has(file, FILE_IS_PRESENT)) {
|
|
/* if has_volatile_hardlinks is true, the nlink value is not reliable */
|
|
if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' with same inode but different attributes: size %" PRIu64 "?%" PRIu64 ", sec %" PRIu64 "?%" PRIu64 ", nsec %d?%d\n",
|
|
file->inode, disk->dir, sub, disk->dir, file->sub,
|
|
file->size, (uint64_t)st->st_size,
|
|
file->mtime_sec, (uint64_t)st->st_mtime,
|
|
file->mtime_nsec, STAT_NSEC(st));
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* LCOV_EXCL_START */
|
|
/* suppose it's hardlink with not synced metadata */
|
|
scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
|
|
return;
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/*
|
|
* Assume a previously used inode, it's the worst case
|
|
* and we handle it removing the duplicate stored inode.
|
|
* If the file is found by name later, it will have the inode restored,
|
|
* otherwise, it will get removed
|
|
*/
|
|
|
|
/* remove from the inode set */
|
|
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
|
|
|
|
/*
|
|
* Clear the inode
|
|
* this is not really needed for correct functionality
|
|
* because we are going to set FILE_IS_WITHOUT_INODE
|
|
* but it's easier for debugging to have invalid inodes set to 0
|
|
*/
|
|
file->inode = 0;
|
|
|
|
/* mark as missing inode */
|
|
file_flag_set(file, FILE_IS_WITHOUT_INODE);
|
|
|
|
/* go further to find it by name */
|
|
}
|
|
|
|
/* initialize for later overwrite */
|
|
is_file_reported = 0;
|
|
is_original_file_size_different_than_zero = 0;
|
|
|
|
/* then try finding it by name */
|
|
file = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub, file_path_hash(sub));
|
|
|
|
/* keep track if the file already exists */
|
|
is_file_already_present = file != 0;
|
|
|
|
if (is_file_already_present) {
|
|
/* if the file is without an inode */
|
|
if (file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
|
|
/* set it now */
|
|
file->inode = st->st_ino;
|
|
|
|
/* insert in the set */
|
|
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
|
|
|
|
/* unmark as missing inode */
|
|
file_flag_clear(file, FILE_IS_WITHOUT_INODE);
|
|
} else {
|
|
/* here the inode has to be different, otherwise we would have found it before */
|
|
if (file->inode == st->st_ino) {
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inconsistency in inode '%" PRIu64 "' for files '%s%s' as unexpected matching\n", file->inode, disk->dir, sub);
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
}
|
|
|
|
/* for sure it cannot be already present */
|
|
if (file_flag_has(file, FILE_IS_PRESENT)) {
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inconsistency in path for file '%s%s' matching and already present\n", disk->dir, sub);
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* check if the file is not changed */
|
|
if (file->size == st->st_size
|
|
&& file->mtime_sec == st->st_mtime
|
|
&& (file->mtime_nsec == STAT_NSEC(st)
|
|
/*
|
|
* Always accept the stored value if it's STAT_NSEC_INVALID
|
|
* it happens when upgrading from an old version of SnapRAID
|
|
* not yet supporting the nanosecond field
|
|
*/
|
|
|| file->mtime_nsec == STAT_NSEC_INVALID
|
|
)
|
|
) {
|
|
/* mark as present */
|
|
file_flag_set(file, FILE_IS_PRESENT);
|
|
|
|
/*
|
|
* Update the nano seconds mtime only if different
|
|
* to avoid unneeded updates
|
|
*/
|
|
if (file->mtime_nsec == STAT_NSEC_INVALID
|
|
&& STAT_NSEC(st) != STAT_NSEC_INVALID
|
|
) {
|
|
file->mtime_nsec = STAT_NSEC(st);
|
|
|
|
/* we have to save the new mtime */
|
|
scan->need_write = 1;
|
|
}
|
|
|
|
/* if when processing the disk we used the past inodes values */
|
|
if (has_past_inodes) {
|
|
/*
|
|
* If persistent inodes are supported, we are sure that the inode number
|
|
* is now different, because otherwise the file would have been found
|
|
* when searching by inode.
|
|
* if the inode is different, it means a rewritten file with the same path
|
|
* like when restoring a backup that restores also the timestamp
|
|
*/
|
|
++scan->count_restore;
|
|
|
|
log_tag("scan:restore:%s:%s\n", disk->name, esc_tag(sub));
|
|
if (is_diff) {
|
|
msg_info("restore %s\n", fmt_term(disk, sub));
|
|
}
|
|
|
|
/* remove from the inode set */
|
|
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
|
|
|
|
/* save the new inode */
|
|
file->inode = st->st_ino;
|
|
|
|
/* reinsert in the inode set */
|
|
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
|
|
|
|
/* we have to save the new inode */
|
|
scan->need_write = 1;
|
|
} else {
|
|
/*
|
|
* Otherwise it's the case of not persistent inode, where doesn't
|
|
* matter if the inode is different or equal, because they have no
|
|
* meaning, and then we don't even save them
|
|
*/
|
|
++scan->count_equal;
|
|
|
|
if (state->opt.gui_verbose) {
|
|
log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub));
|
|
}
|
|
}
|
|
|
|
/* mark the file as kept */
|
|
scan_file_keep(scan, file);
|
|
|
|
/* nothing more to do */
|
|
return;
|
|
}
|
|
|
|
/* here if the file is changed but with the correct name */
|
|
|
|
/* save the info for later printout */
|
|
file_already_present_size = file->size;
|
|
file_already_present_mtime_sec = file->mtime_sec;
|
|
file_already_present_mtime_nsec = file->mtime_nsec;
|
|
|
|
/* keep track if the original file was not of zero size */
|
|
is_original_file_size_different_than_zero = file->size != 0;
|
|
|
|
/* remove it, and continue to insert it again */
|
|
scan_file_remove(scan, file, 1);
|
|
|
|
/* and continue to insert it again */
|
|
} else {
|
|
file_already_present_size = 0;
|
|
file_already_present_mtime_sec = 0;
|
|
file_already_present_mtime_nsec = 0;
|
|
}
|
|
|
|
/*
|
|
* Refresh the info, to ensure that they are synced,
|
|
* note that we refresh only the info of the new or modified files
|
|
* because this is slow operation
|
|
*/
|
|
scan_file_refresh(scan, sub, st, &physical);
|
|
|
|
#ifndef _WIN32
|
|
/*
|
|
* Do a safety check to ensure that the common ext4 case of zeroing
|
|
* the size of a file after a crash doesn't propagate to the backup
|
|
* this check is specific for Linux, so we disable it on Windows
|
|
*/
|
|
if (is_original_file_size_different_than_zero && st->st_size == 0) {
|
|
if (!state->opt.force_zero) {
|
|
/* LCOV_EXCL_START */
|
|
log_error(ESOFT, "The file '%s%s' has unexpected zero size!\n", disk->mount_point, sub);
|
|
log_error(ESOFT, "It's possible that after a kernel crash this file was lost,\n");
|
|
log_error(ESOFT, "and you can use 'snapraid fix -f /%s' to recover it.\n", fmt_poll(disk, sub));
|
|
if (!is_diff) {
|
|
log_fatal(ESOFT, "If this an expected condition you can '%s' anyway using 'snapraid --force-zero %s'\n", state->command, state->command);
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
}
|
|
#else
|
|
/* avoid the unused warning in Windows */
|
|
(void)is_original_file_size_different_than_zero;
|
|
#endif
|
|
|
|
/* insert it */
|
|
file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, STAT_NSEC(st), st->st_ino, physical);
|
|
|
|
/* mark it as present */
|
|
file_flag_set(file, FILE_IS_PRESENT);
|
|
|
|
/*
|
|
* If copy detection is enabled
|
|
* note that the copy detection is tried also for updated files
|
|
* this makes sense because it may happen to have two different copies
|
|
* of the same file, and we move the right one over the wrong one
|
|
* in such case we have a "copy" over an "update"
|
|
*/
|
|
if (!state->opt.force_nocopy) {
|
|
tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
|
|
|
|
/* search for a file with the same name and stamp in all the disks */
|
|
for (i = state->disklist; i != 0; i = i->next) {
|
|
struct snapraid_disk* other_disk = i->data;
|
|
struct snapraid_file* other_file;
|
|
|
|
stamp_lock(other_disk);
|
|
/*
|
|
* If the nanosecond part of the time stamp is valid, search
|
|
* for name and stamp, otherwise for path and stamp
|
|
*/
|
|
if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
|
|
other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
|
|
else
|
|
other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
|
|
stamp_unlock(other_disk);
|
|
|
|
/* if found, and it's a fully hashed file */
|
|
if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
|
|
char path_other[PATH_MAX];
|
|
struct stat other_st;
|
|
|
|
/*
|
|
* Protect the write as multiple threads may write the same FILE_IS_RELOCATED bit.
|
|
*
|
|
* Note that this flag is written in the "shared" portion of the flags,
|
|
* intended exactly to avoid data race from multiple threads using
|
|
* the stamp_lock() protection.
|
|
*/
|
|
stamp_lock(other_disk);
|
|
file_flag_set(other_file, FILE_IS_RELOCATED);
|
|
stamp_unlock(other_disk);
|
|
|
|
/* assume that the file is a copy, and reuse the hash */
|
|
file_copy(other_file, file);
|
|
|
|
/* check if other file still exists */
|
|
pathprint(path_other, sizeof(path_other), "%s%s", other_disk->dir, other_file->sub);
|
|
if (lstat(path_other, &other_st) == 0) {
|
|
++scan->count_copy;
|
|
|
|
log_tag("scan:copy:%s:%s:%s:%s\n", other_disk->name, esc_tag(other_file->sub), disk->name, esc_tag(file->sub));
|
|
if (is_diff) {
|
|
msg_info("copy %s -> %s\n", fmt_term(other_disk, other_file->sub), fmt_term(disk, file->sub));
|
|
}
|
|
} else {
|
|
++scan->count_relocate;
|
|
|
|
log_tag("scan:relocate:%s:%s:%s:%s\n", other_disk->name, esc_tag(other_file->sub), disk->name, esc_tag(file->sub));
|
|
if (is_diff) {
|
|
msg_info("relocate %s -> %s\n", fmt_term(other_disk, other_file->sub), fmt_term(disk, file->sub));
|
|
}
|
|
}
|
|
|
|
/* mark it as reported */
|
|
is_file_reported = 1;
|
|
|
|
/* no need to continue the search */
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* If not yet reported, do it now
|
|
* we postpone this to avoid to print two times the copied files
|
|
*/
|
|
if (!is_file_reported) {
|
|
if (is_file_already_present) {
|
|
++scan->count_change;
|
|
|
|
log_tag("scan:update:%s:%s: %" PRIu64 " %" PRIu64 ".%d -> %" PRIu64 " %" PRIu64 ".%d\n", disk->name, esc_tag(sub),
|
|
file_already_present_size, file_already_present_mtime_sec, file_already_present_mtime_nsec,
|
|
file->size, file->mtime_sec, file->mtime_nsec
|
|
);
|
|
|
|
if (is_diff) {
|
|
msg_info("update %s\n", fmt_term(disk, sub));
|
|
}
|
|
} else {
|
|
++scan->count_insert;
|
|
|
|
log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub));
|
|
if (is_diff) {
|
|
msg_info("add %s\n", fmt_term(disk, sub));
|
|
}
|
|
}
|
|
}
|
|
|
|
/* insert the file in the delayed list */
|
|
scan_file_insert(scan, file);
|
|
}
|
|
|
|
/**
|
|
* Remove the specified dir from the data set.
|
|
*/
|
|
static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
|
|
{
|
|
struct snapraid_disk* disk = scan->disk;
|
|
|
|
/* state changed */
|
|
scan->need_write = 1;
|
|
|
|
/* remove the file from the dir containers */
|
|
tommy_hashdyn_remove_existing(&disk->dirset, &dir->nodeset);
|
|
tommy_list_remove_existing(&disk->dirlist, &dir->nodelist);
|
|
|
|
/* deallocate */
|
|
dir_free(dir);
|
|
}
|
|
|
|
/**
|
|
* Insert the specified dir in the data set.
|
|
*/
|
|
static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
|
|
{
|
|
struct snapraid_disk* disk = scan->disk;
|
|
|
|
/* state changed */
|
|
scan->need_write = 1;
|
|
|
|
/* insert the dir in the dir containers */
|
|
tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
|
|
tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
|
|
}
|
|
|
|
/**
|
|
* Process a dir.
|
|
*/
|
|
static void scan_emptydir(struct snapraid_scan* scan, const char* sub)
|
|
{
|
|
struct snapraid_disk* disk = scan->disk;
|
|
struct snapraid_dir* dir;
|
|
|
|
/* check if the dir already exists */
|
|
dir = tommy_hashdyn_search(&disk->dirset, dir_name_compare, sub, dir_name_hash(sub));
|
|
if (dir) {
|
|
/* check if multiple files have the same name */
|
|
if (dir_flag_has(dir, FILE_IS_PRESENT)) {
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inconsistency for dir '%s%s'\n", disk->dir, sub);
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* mark as present */
|
|
dir_flag_set(dir, FILE_IS_PRESENT);
|
|
|
|
/* nothing more to do */
|
|
return;
|
|
} else {
|
|
/* and continue to insert it */
|
|
}
|
|
|
|
/* insert it */
|
|
dir = dir_alloc(sub);
|
|
|
|
/* mark it as present */
|
|
dir_flag_set(dir, FILE_IS_PRESENT);
|
|
|
|
/* insert it in the delayed insert list */
|
|
tommy_list_insert_tail(&scan->dir_insert_list, &dir->nodelist, dir);
|
|
}
|
|
|
|
struct dirent_sorted {
|
|
/* node for data structures */
|
|
tommy_node node;
|
|
|
|
#if HAVE_STRUCT_DIRENT_D_INO
|
|
uint64_t d_ino; /**< Inode number. */
|
|
#endif
|
|
#if HAVE_STRUCT_DIRENT_D_TYPE
|
|
uint32_t d_type; /**< File type. */
|
|
#endif
|
|
#if HAVE_STRUCT_DIRENT_D_STAT
|
|
struct stat d_stat; /**< Stat result. */
|
|
#endif
|
|
char d_name[]; /**< Variable length name. It must be the last field. */
|
|
};
|
|
|
|
#if HAVE_STRUCT_DIRENT_D_INO
|
|
static int dd_ino_compare(const void* void_a, const void* void_b)
|
|
{
|
|
const struct dirent_sorted* a = void_a;
|
|
const struct dirent_sorted* b = void_b;
|
|
|
|
if (a->d_ino < b->d_ino)
|
|
return -1;
|
|
if (a->d_ino > b->d_ino)
|
|
return 1;
|
|
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
static int dd_name_compare(const void* void_a, const void* void_b)
|
|
{
|
|
const struct dirent_sorted* a = void_a;
|
|
const struct dirent_sorted* b = void_b;
|
|
|
|
return strcmp(a->d_name, b->d_name);
|
|
}
|
|
|
|
/**
|
|
* Return the stat info of a dir entry.
|
|
*/
|
|
#if HAVE_STRUCT_DIRENT_D_STAT
|
|
#define DSTAT(file, dd, buf) dstat(dd)
|
|
struct stat* dstat(struct dirent_sorted* dd)
|
|
{
|
|
return &dd->d_stat;
|
|
}
|
|
#else
|
|
#define DSTAT(file, dd, buf) dstat(disk, file, buf)
|
|
struct stat* dstat(struct snapraid_disk* disk, const char* file, struct stat* st)
|
|
{
|
|
if (lstat(file, st) != 0) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: Stat error. %s.\n", es(errno), 0, disk->name, esc_tag(file), strerror(errno));
|
|
log_fatal(errno, "Error in stat file/directory '%s'. %s.\n", file, strerror(errno));
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
return st;
|
|
}
|
|
#endif
|
|
|
|
/**
|
|
* Process a directory.
|
|
* Return != 0 if at least one file or link is processed.
|
|
*/
|
|
static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* path_next, char* sub_next, char* tmp)
|
|
{
|
|
struct snapraid_state* state = scan->state;
|
|
struct snapraid_disk* disk = scan->disk;
|
|
int processed = 0;
|
|
DIR* d;
|
|
tommy_list list;
|
|
tommy_node* node;
|
|
size_t path_len;
|
|
size_t sub_len;
|
|
|
|
path_len = strlen(path_next);
|
|
sub_len = strlen(sub_next);
|
|
|
|
tommy_list_init(&list);
|
|
|
|
d = opendir(path_next);
|
|
if (!d) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: Open dir error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno));
|
|
log_fatal(errno, "Error opening directory '%s'. %s.\n", path_next, strerror(errno));
|
|
if (level == 0)
|
|
log_fatal(errno, "If this is the disk mount point, remember to create it manually\n");
|
|
else
|
|
log_fatal(errno, "If it's a permission problem, you can exclude it in the config file with: exclude /%s\n", sub_next);
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* read the full directory */
|
|
while (1) {
|
|
struct dirent_sorted* entry;
|
|
const char* name;
|
|
struct dirent* dd;
|
|
size_t name_len;
|
|
|
|
/*
|
|
* Clear errno to differentiate the end of the stream and an error condition
|
|
*
|
|
* From the Linux readdir() manpage:
|
|
* "If the end of the directory stream is reached, NULL is returned and errno is not changed.
|
|
* If an error occurs, NULL is returned and errno is set appropriately."
|
|
*/
|
|
errno = 0;
|
|
dd = readdir(d);
|
|
if (dd == 0 && errno != 0) {
|
|
/* LCOV_EXCL_START */
|
|
/* restore removing additions */
|
|
path_next[path_len] = 0;
|
|
sub_next[sub_len] = 0;
|
|
log_tag("%s:%u:%s:%s: Read dir error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno));
|
|
log_fatal(errno, "Error reading directory '%s'. %s.\n", path_next, strerror(errno));
|
|
log_fatal(errno, "You can exclude it in the config file with: exclude /%s\n", sub_next);
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
if (dd == 0) {
|
|
break; /* finished */
|
|
}
|
|
|
|
/* skip "." and ".." files */
|
|
name = dd->d_name;
|
|
if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
|
|
continue;
|
|
|
|
pathcatl(path_next, path_len, PATH_MAX, name);
|
|
|
|
/* check for not supported file names */
|
|
if (name[0] == 0) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: Unsupported name error.\n", es(ESOFT), 0, disk->name, esc_tag(path_next));
|
|
log_fatal(ESOFT, "Unsupported name '%s' in file '%s'.\n", name, path_next);
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* exclude hidden files even before calling lstat() */
|
|
if (filter_hidden(state->filter_hidden, dd) != 0) {
|
|
msg_verbose("Excluding hidden '%s'\n", path_next);
|
|
continue;
|
|
}
|
|
|
|
/* exclude content files even before calling lstat() */
|
|
if (filter_content(&state->contentlist, path_next) != 0) {
|
|
msg_verbose("Excluding content '%s'\n", path_next);
|
|
continue;
|
|
}
|
|
|
|
name_len = strlen(dd->d_name);
|
|
entry = malloc_nofail(sizeof(struct dirent_sorted) + name_len + 1);
|
|
|
|
/* copy the dir entry */
|
|
#if HAVE_STRUCT_DIRENT_D_INO
|
|
entry->d_ino = dd->d_ino;
|
|
#endif
|
|
#if HAVE_STRUCT_DIRENT_D_TYPE
|
|
entry->d_type = dd->d_type;
|
|
#endif
|
|
#if HAVE_STRUCT_DIRENT_D_STAT
|
|
/* convert dirent to lstat result */
|
|
dirent_lstat(dd, &entry->d_stat);
|
|
|
|
/* note that at this point the st_mode may be 0 */
|
|
#endif
|
|
memcpy(entry->d_name, dd->d_name, name_len + 1);
|
|
|
|
/* insert in the list */
|
|
tommy_list_insert_tail(&list, &entry->node, entry);
|
|
|
|
/* process ignore files */
|
|
if (strcmp(".snapraidignore", dd->d_name) == 0)
|
|
state_load_ignore_file(&scan->local_filter_list, path_next, sub_next);
|
|
}
|
|
|
|
if (closedir(d) != 0) {
|
|
/* LCOV_EXCL_START */
|
|
/* restore removing additions */
|
|
path_next[path_len] = 0;
|
|
log_tag("%s:%u:%s:%s: Close dir error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno));
|
|
log_fatal(errno, "Error closing directory '%s'. %s.\n", path_next, strerror(errno));
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
if (state->opt.force_order == SORT_ALPHA) {
|
|
/*
|
|
* If requested sort alphabetically
|
|
* this is mainly done for testing to ensure to always
|
|
* process in the same way in different platforms
|
|
*/
|
|
tommy_list_sort(&list, dd_name_compare);
|
|
}
|
|
#if HAVE_STRUCT_DIRENT_D_INO
|
|
else if (!disk->has_volatile_inodes) {
|
|
/*
|
|
* If inodes are persistent
|
|
* sort the list of dir entries by inodes
|
|
*/
|
|
tommy_list_sort(&list, dd_ino_compare);
|
|
}
|
|
/* otherwise just keep the insertion order */
|
|
#endif
|
|
|
|
/* process the sorted dir entries */
|
|
node = list;
|
|
while (node != 0) {
|
|
struct snapraid_filter* reason = 0;
|
|
struct dirent_sorted* dd = node->data;
|
|
const char* name = dd->d_name;
|
|
struct stat* st;
|
|
int type;
|
|
#if !HAVE_STRUCT_DIRENT_D_STAT
|
|
struct stat st_buf;
|
|
#endif
|
|
|
|
pathcatl(path_next, path_len, PATH_MAX, name);
|
|
pathcatl(sub_next, sub_len, PATH_MAX, name);
|
|
|
|
/* start with an unknown type */
|
|
type = -1;
|
|
st = 0;
|
|
|
|
/* if dirent has the type, use it */
|
|
#if HAVE_STRUCT_DIRENT_D_TYPE
|
|
switch (dd->d_type) {
|
|
case DT_UNKNOWN : break;
|
|
case DT_REG : type = 0; break;
|
|
case DT_LNK : type = 1; break;
|
|
case DT_DIR : type = 2; break;
|
|
default : type = 3; break;
|
|
}
|
|
#endif
|
|
|
|
/* if type is still unknown */
|
|
if (type < 0) {
|
|
/* get the type from stat */
|
|
st = DSTAT(path_next, dd, &st_buf);
|
|
|
|
#if HAVE_STRUCT_DIRENT_D_STAT
|
|
/*
|
|
* If the st_mode field is missing, takes care to fill it using normal lstat()
|
|
* at now this can happen only in Windows (with HAVE_STRUCT_DIRENT_D_STAT defined),
|
|
* because we use a directory reading method that doesn't read info about ReparsePoint.
|
|
* Note that here we cannot call here lstat_sync(), because we don't know what kind
|
|
* of file is it, and lstat_sync() doesn't always work
|
|
*/
|
|
if (st->st_mode == 0) {
|
|
if (lstat(path_next, st) != 0) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: Stat error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno));
|
|
log_fatal(errno, "Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
}
|
|
#endif
|
|
|
|
if (S_ISREG(st->st_mode))
|
|
type = 0;
|
|
else if (S_ISLNK(st->st_mode))
|
|
type = 1;
|
|
else if (S_ISDIR(st->st_mode))
|
|
type = 2;
|
|
else
|
|
type = 3;
|
|
}
|
|
|
|
if (type == 0) { /* REG */
|
|
if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0
|
|
&& filter_path(&scan->local_filter_list, &reason, disk->name, sub_next) == 0) {
|
|
|
|
/* late stat, if not yet called */
|
|
if (!st)
|
|
st = DSTAT(path_next, dd, &st_buf);
|
|
|
|
#if HAVE_LSTAT_SYNC
|
|
/*
|
|
* If the st_ino field is missing, takes care to fill it using the extended lstat()
|
|
* this can happen only in Windows
|
|
*/
|
|
if (st->st_ino == 0 || st->st_nlink == 0) {
|
|
if (lstat_sync(path_next, st, 0) != 0) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: Stat error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno));
|
|
log_fatal(errno, "Error in stat file '%s'. %s.\n", path_next, strerror(errno));
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
}
|
|
#endif
|
|
|
|
scan_file(scan, is_diff, sub_next, st, FILEPHY_UNREAD_OFFSET);
|
|
processed = 1;
|
|
} else {
|
|
msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
|
|
}
|
|
} else if (type == 1) { /* LNK */
|
|
if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0
|
|
&& filter_path(&scan->local_filter_list, &reason, disk->name, sub_next) == 0) {
|
|
int ret;
|
|
|
|
ret = readlink(path_next, tmp, PATH_MAX);
|
|
if (ret >= PATH_MAX) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: Readlink error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno));
|
|
log_fatal(EINTERNAL, "Error in readlink file '%s'. Symlink too long.\n", path_next);
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
if (ret < 0) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: Readlink error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno));
|
|
log_fatal(errno, "Error in readlink file '%s'. %s.\n", path_next, strerror(errno));
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
if (ret == 0)
|
|
log_error(ESOFT, "WARNING! Empty symbolic link '%s'.\n", path_next);
|
|
|
|
/* readlink doesn't put the final 0 */
|
|
tmp[ret] = 0;
|
|
|
|
/* process as a symbolic link */
|
|
scan_link(scan, is_diff, sub_next, tmp, FILE_IS_SYMLINK);
|
|
processed = 1;
|
|
} else {
|
|
msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
|
|
}
|
|
} else if (type == 2) { /* DIR */
|
|
if (filter_subdir(&state->filterlist, &reason, disk->name, sub_next) == 0
|
|
&& filter_subdir(&scan->local_filter_list, &reason, disk->name, sub_next) == 0) {
|
|
#ifndef _WIN32
|
|
/* late stat, if not yet called */
|
|
if (!st)
|
|
st = DSTAT(path_next, dd, &st_buf);
|
|
|
|
/*
|
|
* In Unix don't follow mount points in different devices
|
|
* in Windows we are already skipping them reporting them as special files
|
|
*/
|
|
if ((uint64_t)st->st_dev != disk->dir_device) {
|
|
log_tag("%s:%u:%s:%s: Ignoring mount point.\n", es(ESOFT), 0, disk->name, esc_tag(path_next));
|
|
log_error(ESOFT, "WARNING! Ignoring mount point '%s' because it appears to be in a different device\n", path_next);
|
|
} else
|
|
#endif
|
|
{
|
|
/* recurse */
|
|
pathslash(path_next, PATH_MAX);
|
|
pathslash(sub_next, PATH_MAX);
|
|
if (scan_sub(scan, level + 1, is_diff, path_next, sub_next, tmp) == 0) {
|
|
/* restore removing additions */
|
|
pathcatl(sub_next, sub_len, PATH_MAX, name);
|
|
/* scan the directory as empty dir */
|
|
scan_emptydir(scan, sub_next);
|
|
}
|
|
/* or we processed something internally, or we have added the empty dir */
|
|
processed = 1;
|
|
}
|
|
} else {
|
|
msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
|
|
}
|
|
} else {
|
|
if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0
|
|
&& filter_path(&scan->local_filter_list, &reason, disk->name, sub_next) == 0) {
|
|
/* late stat, if not yet called */
|
|
if (!st)
|
|
st = DSTAT(path_next, dd, &st_buf);
|
|
|
|
log_tag("%s:%u:%s:%s: Ignoring special file.\n", es(ESOFT), 0, disk->name, esc_tag(path_next));
|
|
log_error(ESOFT, "WARNING! Ignoring special '%s' file '%s'\n", stat_desc(st), path_next);
|
|
} else {
|
|
msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
|
|
}
|
|
}
|
|
|
|
/* next entry */
|
|
node = node->next;
|
|
|
|
/* free the present one */
|
|
free(dd);
|
|
}
|
|
|
|
return processed;
|
|
}
|
|
|
|
/**
|
|
* Process a directory.
|
|
* Return != 0 if at least one file or link is processed.
|
|
*/
|
|
static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const char* dir, const char* sub)
|
|
{
|
|
/* working buffers used by scan_sub() */
|
|
char path_next[PATH_MAX];
|
|
char sub_next[PATH_MAX];
|
|
char tmp[PATH_MAX];
|
|
|
|
pathcpy(path_next, sizeof(path_next), dir);
|
|
pathcpy(sub_next, sizeof(sub_next), sub);
|
|
|
|
return scan_sub(scan, level, is_diff, path_next, sub_next, tmp);
|
|
}
|
|
|
|
static void* scan_disk(void* arg)
|
|
{
|
|
struct snapraid_scan* scan = arg;
|
|
struct snapraid_disk* disk = scan->disk;
|
|
int ret;
|
|
int has_persistent_inodes;
|
|
int has_syncronized_hardlinks;
|
|
uint64_t start;
|
|
|
|
/* check if the disk supports persistent inodes */
|
|
ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0, 0, 0, 0, 0);
|
|
if (ret < 0) {
|
|
/* LCOV_EXCL_START */
|
|
log_tag("%s:%u:%s:%s: Filesystem info error. %s.\n", es(errno), 0, disk->name, esc_tag(disk->dir), strerror(errno));
|
|
log_fatal(errno, "Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
|
|
exit(EXIT_FAILURE);
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
if (!has_persistent_inodes) {
|
|
disk->has_volatile_inodes = 1;
|
|
}
|
|
if (!has_syncronized_hardlinks) {
|
|
disk->has_volatile_hardlinks = 1;
|
|
}
|
|
|
|
/* if inodes or UUID are not persistent/changed/unsupported */
|
|
if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
|
|
/*
|
|
* Remove all the inodes from the inode collection
|
|
* if they are not persistent, all of them could be changed now
|
|
* and we don't want to find false matching ones
|
|
* See scan_file() for more details
|
|
*/
|
|
tommy_node* node = disk->filelist;
|
|
while (node) {
|
|
struct snapraid_file* file = node->data;
|
|
|
|
node = node->next;
|
|
|
|
/* remove from the inode set */
|
|
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
|
|
|
|
/* clear the inode */
|
|
file->inode = 0;
|
|
|
|
/* mark as missing inode */
|
|
file_flag_set(file, FILE_IS_WITHOUT_INODE);
|
|
}
|
|
}
|
|
|
|
start = os_tick_ms();
|
|
|
|
scan_dir(scan, 0, scan->is_diff, disk->dir, "");
|
|
|
|
if (!scan->is_diff)
|
|
msg_progress("Scanned %s in %" PRIu64 " seconds\n", disk->name, (os_tick_ms() - start) / 1000);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int state_diffscan(struct snapraid_state* state, int is_diff)
|
|
{
|
|
tommy_node* i;
|
|
tommy_node* j;
|
|
tommy_list scanlist;
|
|
int done;
|
|
msg_ptr* msg;
|
|
struct snapraid_scan total;
|
|
int no_difference;
|
|
tommy_list_init(&scanlist);
|
|
|
|
if (is_diff)
|
|
msg_progress("Comparing...\n");
|
|
else
|
|
msg_progress("Scanning...\n");
|
|
|
|
log_tag("list:scan_begin\n");
|
|
|
|
/* allocate all the scan data */
|
|
for (i = state->disklist; i != 0; i = i->next) {
|
|
struct snapraid_disk* disk = i->data;
|
|
struct snapraid_scan* scan;
|
|
|
|
scan = scan_alloc(state, disk, is_diff);
|
|
|
|
tommy_list_insert_tail(&scanlist, &scan->node, scan);
|
|
}
|
|
|
|
/* first scan all the directory and find new and deleted files */
|
|
for (i = scanlist; i != 0; i = i->next) {
|
|
struct snapraid_scan* scan = i->data;
|
|
#if HAVE_THREAD
|
|
if (state->opt.skip_multi_scan)
|
|
scan_disk(scan);
|
|
else
|
|
thread_create(&scan->thread, scan_disk, scan);
|
|
#else
|
|
scan_disk(scan);
|
|
#endif
|
|
}
|
|
|
|
#if HAVE_THREAD
|
|
/* wait for all threads to terminate */
|
|
for (i = scanlist; i != 0; i = i->next) {
|
|
struct snapraid_scan* scan = i->data;
|
|
void* retval;
|
|
|
|
/* wait for thread termination */
|
|
if (!state->opt.skip_multi_scan)
|
|
thread_join(scan->thread, &retval);
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* We split the search in two phases because to detect files
|
|
* moved from one disk to another we have to start deletion
|
|
* only when all disks have all the new files found
|
|
*/
|
|
|
|
/* now process all the new and deleted files */
|
|
for (i = scanlist; i != 0; i = i->next) {
|
|
struct snapraid_scan* scan = i->data;
|
|
struct snapraid_disk* disk = scan->disk;
|
|
tommy_node* node;
|
|
unsigned phy_dup;
|
|
uint64_t phy_last;
|
|
struct snapraid_file* phy_file_last;
|
|
|
|
/* check for removed files */
|
|
node = disk->filelist;
|
|
while (node) {
|
|
struct snapraid_file* file = node->data;
|
|
|
|
/* next node */
|
|
node = node->next;
|
|
|
|
/* remove if not present */
|
|
if (!file_flag_has(file, FILE_IS_PRESENT)) {
|
|
/* here we are in mono-thread context, no need to use the stamp_lock() to read FILE_IS_RELOCATED */
|
|
if (!file_flag_has(file, FILE_IS_RELOCATED)) {
|
|
++scan->count_remove;
|
|
|
|
log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(file->sub));
|
|
if (is_diff) {
|
|
msg_info("remove %s\n", fmt_term(disk, file->sub));
|
|
}
|
|
}
|
|
|
|
scan_file_remove(scan, file, 1);
|
|
}
|
|
}
|
|
|
|
/* check for removed links */
|
|
node = disk->linklist;
|
|
while (node) {
|
|
struct snapraid_link* slink = node->data;
|
|
|
|
/* next node */
|
|
node = node->next;
|
|
|
|
/* remove if not present */
|
|
if (!link_flag_has(slink, FILE_IS_PRESENT)) {
|
|
++scan->count_remove;
|
|
|
|
log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(slink->sub));
|
|
if (is_diff) {
|
|
msg_info("remove %s\n", fmt_term(disk, slink->sub));
|
|
}
|
|
|
|
scan_link_remove(scan, slink);
|
|
}
|
|
}
|
|
|
|
/* check for removed dirs */
|
|
node = disk->dirlist;
|
|
while (node) {
|
|
struct snapraid_dir* dir = node->data;
|
|
|
|
/* next node */
|
|
node = node->next;
|
|
|
|
/* remove if not present */
|
|
if (!dir_flag_has(dir, FILE_IS_PRESENT)) {
|
|
scan_emptydir_remove(scan, dir);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Sort the files before inserting them
|
|
* we use a stable sort to ensure that if the reported physical offset/inode
|
|
* are always 0, we keep at least the directory order
|
|
*/
|
|
switch (state->opt.force_order) {
|
|
case SORT_PHYSICAL :
|
|
tommy_list_sort(&scan->file_insert_list, file_physical_compare);
|
|
break;
|
|
case SORT_INODE :
|
|
tommy_list_sort(&scan->file_insert_list, file_inode_compare);
|
|
break;
|
|
case SORT_ALPHA :
|
|
tommy_list_sort(&scan->file_insert_list, file_path_compare);
|
|
break;
|
|
case SORT_DIR :
|
|
/* already in order */
|
|
break;
|
|
}
|
|
|
|
/*
|
|
* Insert all the new files, we insert them only after the deletion
|
|
* to reuse the just freed space
|
|
* also check if the physical offset reported are fakes or not
|
|
*/
|
|
node = scan->file_insert_list;
|
|
phy_dup = 0;
|
|
phy_last = FILEPHY_UNREAD_OFFSET;
|
|
phy_file_last = 0;
|
|
while (node) {
|
|
struct snapraid_file* file = node->data;
|
|
|
|
/* if the file is not empty, count duplicate physical offsets */
|
|
if (state->opt.force_order == SORT_PHYSICAL && file->size != 0) {
|
|
if (phy_file_last != 0 && file->physical == phy_last
|
|
/* files without offset are expected to have duplicates */
|
|
&& phy_last != FILEPHY_WITHOUT_OFFSET
|
|
) {
|
|
/*
|
|
* If verbose, print the list of duplicates real offsets
|
|
* other cases are for offsets not supported, so we don't need to report them file by file
|
|
*/
|
|
if (phy_last >= FILEPHY_REAL_OFFSET) {
|
|
log_error(ESOFT, "WARNING! Files '%s%s' and '%s%s' share the same physical offset %" PRId64 ".\n", disk->mount_point, phy_file_last->sub, disk->mount_point, file->sub, phy_last);
|
|
}
|
|
++phy_dup;
|
|
}
|
|
phy_file_last = file;
|
|
phy_last = file->physical;
|
|
}
|
|
|
|
/* next node */
|
|
node = node->next;
|
|
|
|
/* insert in the parity */
|
|
scan_file_allocate(scan, file);
|
|
}
|
|
|
|
/*
|
|
* Mark the disk without reliable physical offset if it has duplicates
|
|
* here it should never happen because we already sorted out hardlinks
|
|
*/
|
|
if (state->opt.force_order == SORT_PHYSICAL && phy_dup > 0) {
|
|
disk->has_unreliable_physical = 1;
|
|
}
|
|
|
|
/* insert all the new links */
|
|
node = scan->link_insert_list;
|
|
while (node) {
|
|
struct snapraid_link* slink = node->data;
|
|
|
|
/* next node */
|
|
node = node->next;
|
|
|
|
/* insert it */
|
|
scan_link_insert(scan, slink);
|
|
}
|
|
|
|
/* insert all the new dirs */
|
|
node = scan->dir_insert_list;
|
|
while (node) {
|
|
struct snapraid_dir* dir = node->data;
|
|
|
|
/* next node */
|
|
node = node->next;
|
|
|
|
/* insert it */
|
|
scan_emptydir_insert(scan, dir);
|
|
}
|
|
}
|
|
|
|
/* propagate the state change (after all the scan operations are called) */
|
|
for (i = scanlist; i != 0; i = i->next) {
|
|
struct snapraid_scan* scan = i->data;
|
|
if (scan->need_write) {
|
|
state->need_write = 1;
|
|
}
|
|
}
|
|
|
|
/* check for disks where all the previously existing files where removed */
|
|
if (!state->opt.force_empty) {
|
|
int all_missing = 0;
|
|
int all_rewritten = 0;
|
|
done = 0;
|
|
for (i = state->disklist, j = scanlist; i != 0; i = i->next, j = j->next) {
|
|
struct snapraid_disk* disk = i->data;
|
|
struct snapraid_scan* scan = j->data;
|
|
|
|
if (scan->count_equal == 0
|
|
&& scan->count_move == 0
|
|
&& scan->count_restore == 0
|
|
&& (scan->count_remove != 0 || scan->count_change != 0)
|
|
) {
|
|
if (!done) {
|
|
done = 1;
|
|
log_error(ESOFT, "WARNING! All the files previously present in disk '%s' at dir '%s'", disk->name, disk->mount_point);
|
|
} else {
|
|
log_error(ESOFT, ", disk '%s' at dir '%s'", disk->name, disk->mount_point);
|
|
}
|
|
|
|
/* detect the special condition of all files missing */
|
|
if (scan->count_change == 0)
|
|
all_missing = 1;
|
|
|
|
/* detect the special condition of all files rewritten */
|
|
if (scan->count_remove == 0)
|
|
all_rewritten = 1;
|
|
}
|
|
}
|
|
if (done) {
|
|
log_fatal(ESOFT, "\nare now missing or have been rewritten!\n");
|
|
if (all_rewritten) {
|
|
log_error(ESOFT, "This could occur when restoring a disk from a backup\n");
|
|
log_error(ESOFT, "program that is not setting correctly the timestamps.\n");
|
|
}
|
|
if (all_missing) {
|
|
log_error(ESOFT, "This could occur when some disks are not mounted\n");
|
|
log_error(ESOFT, "in the expected directory.\n");
|
|
}
|
|
if (!is_diff) {
|
|
log_fatal(ESOFT, "If you want to '%s' anyway, use 'snapraid --force-empty %s'.\n", state->command, state->command);
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
}
|
|
|
|
/* check for disks without the physical offset support */
|
|
if (state->opt.force_order == SORT_PHYSICAL) {
|
|
done = 0;
|
|
for (i = state->disklist; i != 0; i = i->next) {
|
|
struct snapraid_disk* disk = i->data;
|
|
|
|
if (disk->has_unreliable_physical) {
|
|
if (!done) {
|
|
done = 1;
|
|
log_error(ESOFT, "WARNING! Physical offsets not supported for disk '%s'", disk->name);
|
|
} else {
|
|
log_error(ESOFT, ", '%s'", disk->name);
|
|
}
|
|
}
|
|
}
|
|
if (done) {
|
|
log_error(ESOFT, ". The order of files won't be optimal.\n");
|
|
}
|
|
}
|
|
|
|
/* check for disks without persistent inodes */
|
|
done = 0;
|
|
for (i = state->disklist; i != 0; i = i->next) {
|
|
struct snapraid_disk* disk = i->data;
|
|
|
|
if (disk->has_volatile_inodes) {
|
|
if (!done) {
|
|
done = 1;
|
|
log_error(ESOFT, "WARNING! Inodes are not persistent for disks: '%s'", disk->name);
|
|
} else {
|
|
log_error(ESOFT, ", '%s'", disk->name);
|
|
}
|
|
}
|
|
}
|
|
if (done) {
|
|
log_error(ESOFT, ". Inodes are not used to detect move operations.\n");
|
|
}
|
|
|
|
/* check for disks with changed UUID */
|
|
done = 0;
|
|
for (i = state->disklist; i != 0; i = i->next) {
|
|
struct snapraid_disk* disk = i->data;
|
|
|
|
/*
|
|
* Don't print the message if the UUID changed because before
|
|
* it was no set.
|
|
* This is the normal condition for an empty disk because it
|
|
* isn't stored
|
|
*/
|
|
if (disk->has_different_uuid && !disk->had_empty_uuid) {
|
|
if (!done) {
|
|
done = 1;
|
|
log_error(ESOFT, "WARNING! UUID is changed for disks: '%s'", disk->name);
|
|
} else {
|
|
log_error(ESOFT, ", '%s'", disk->name);
|
|
}
|
|
}
|
|
}
|
|
if (done) {
|
|
log_error(ESOFT, ". Inodes are not used to detect move operations.\n");
|
|
}
|
|
|
|
/* check for disks with unsupported UUID */
|
|
done = 0;
|
|
for (i = state->disklist; i != 0; i = i->next) {
|
|
struct snapraid_disk* disk = i->data;
|
|
|
|
if (disk->has_unsupported_uuid) {
|
|
if (!done) {
|
|
done = 1;
|
|
log_error(ESOFT, "WARNING! UUID is unsupported for disks: '%s'", disk->name);
|
|
} else {
|
|
log_error(ESOFT, ", '%s'", disk->name);
|
|
}
|
|
}
|
|
}
|
|
if (done) {
|
|
log_error(ESOFT, ". Not using inodes to detect move operations.\n");
|
|
#if defined(_linux) && !HAVE_BLKID
|
|
log_error(ESOFT, "The 'blkid' library is not linked in SnapRAID!\n");
|
|
log_error(ESOFT, "Try rebuilding it after installing the libblkid-dev or libblkid-devel package.\n");
|
|
#endif
|
|
}
|
|
|
|
total.count_equal = 0;
|
|
total.count_move = 0;
|
|
total.count_copy = 0;
|
|
total.count_relocate = 0;
|
|
total.count_restore = 0;
|
|
total.count_change = 0;
|
|
total.count_remove = 0;
|
|
total.count_insert = 0;
|
|
|
|
for (i = scanlist; i != 0; i = i->next) {
|
|
struct snapraid_scan* scan = i->data;
|
|
total.count_equal += scan->count_equal;
|
|
total.count_move += scan->count_move;
|
|
total.count_copy += scan->count_copy;
|
|
total.count_relocate += scan->count_relocate;
|
|
total.count_restore += scan->count_restore;
|
|
total.count_change += scan->count_change;
|
|
total.count_remove += scan->count_remove;
|
|
total.count_insert += scan->count_insert;
|
|
}
|
|
|
|
if (is_diff) {
|
|
msg_status("\n");
|
|
msg = msg_status;
|
|
} else {
|
|
msg = msg_verbose;
|
|
}
|
|
|
|
msg("%8u equal\n", total.count_equal);
|
|
msg("%8u added\n", total.count_insert);
|
|
msg("%8u removed\n", total.count_remove);
|
|
msg("%8u updated\n", total.count_change);
|
|
msg("%8u moved\n", total.count_move);
|
|
msg("%8u copied\n", total.count_copy);
|
|
msg("%8u relocated\n", total.count_relocate);
|
|
msg("%8u restored\n", total.count_restore);
|
|
|
|
log_tag("summary:equal:%u\n", total.count_equal);
|
|
log_tag("summary:added:%u\n", total.count_insert);
|
|
log_tag("summary:removed:%u\n", total.count_remove);
|
|
log_tag("summary:updated:%u\n", total.count_change);
|
|
log_tag("summary:moved:%u\n", total.count_move);
|
|
log_tag("summary:copied:%u\n", total.count_copy);
|
|
log_tag("summary:relocated:%u\n", total.count_relocate);
|
|
log_tag("summary:restored:%u\n", total.count_restore);
|
|
log_tag("list:scan_end\n");
|
|
|
|
/* save in the state */
|
|
state->removed_files = total.count_remove;
|
|
state->updated_files = total.count_change;
|
|
|
|
no_difference = !total.count_move && !total.count_copy && !total.count_relocate && !total.count_restore
|
|
&& !total.count_change && !total.count_remove && !total.count_insert;
|
|
|
|
if (is_diff) {
|
|
if (!no_difference) {
|
|
msg_status("There are differences!\n");
|
|
} else {
|
|
msg_status("No differences\n");
|
|
}
|
|
if (state->unsynced_blocks != 0)
|
|
log_error(EUSER, "The last sync was interrupted. Run it again!\n");
|
|
|
|
if (state->unsynced_blocks != 0) {
|
|
log_tag("summary:exit:unsynced\n");
|
|
} else if (!no_difference) {
|
|
log_tag("summary:exit:diff\n");
|
|
} else {
|
|
log_tag("summary:exit:equal\n");
|
|
}
|
|
}
|
|
|
|
log_flush();
|
|
|
|
tommy_list_foreach(&scanlist, (tommy_foreach_func*)scan_free);
|
|
|
|
/* check the file-system on all disks */
|
|
state_fscheck(state, "after scan");
|
|
|
|
if (is_diff) {
|
|
/* check for file difference */
|
|
if (!no_difference)
|
|
return 1;
|
|
|
|
/* check also for incomplete "sync" */
|
|
if (state->unsynced_blocks != 0)
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int state_diff(struct snapraid_state* state)
|
|
{
|
|
return state_diffscan(state, 1);
|
|
}
|
|
|
|
void state_scan(struct snapraid_state* state)
|
|
{
|
|
(void)state_diffscan(state, 0); /* ignore return value */
|
|
}
|
|
|