LCOV - code coverage report
Current view: top level - cmdline - scan.c (source / functions) Hit Total Coverage
Test: lcov.info Lines: 516 618 83.5 %
Date: 2015-05-12 Functions: 22 22 100.0 %

          Line data    Source code
       1             : /*
       2             :  * Copyright (C) 2011 Andrea Mazzoleni
       3             :  *
       4             :  * This program is free software: you can redistribute it and/or modify
       5             :  * it under the terms of the GNU General Public License as published by
       6             :  * the Free Software Foundation, either version 3 of the License, or
       7             :  * (at your option) any later version.
       8             :  *
       9             :  * This program is distributed in the hope that it will be useful,
      10             :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      12             :  * GNU General Public License for more details.
      13             :  *
      14             :  * You should have received a copy of the GNU General Public License
      15             :  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "portable.h"
      19             : 
      20             : #include "support.h"
      21             : #include "elem.h"
      22             : #include "state.h"
      23             : #include "parity.h"
      24             : 
      25             : struct snapraid_scan {
      26             :         struct snapraid_state* state; /**< State used. */
      27             :         struct snapraid_disk* disk; /**< Disk used. */
      28             : 
      29             :         /**
      30             :          * Counters of changes.
      31             :          */
      32             :         unsigned count_equal; /**< Files equal. */
      33             :         unsigned count_move; /**< Files with a different name, but equal inode, size and timestamp in the same disk. */
      34             :         unsigned count_restore; /**< Files with equal name, size and timestamp, but different inode. */
      35             :         unsigned count_change; /**< Files with same name, but different size and/or timestamp. */
      36             :         unsigned count_copy; /**< Files new, with same name size and timestamp of a file in a different disk. */
      37             :         unsigned count_insert; /**< Files new. */
      38             :         unsigned count_remove; /**< Files removed. */
      39             : 
      40             :         tommy_list file_insert_list; /**< Files to insert. */
      41             :         tommy_list link_insert_list; /**< Links to insert. */
      42             :         tommy_list dir_insert_list; /**< Dirs to insert. */
      43             : 
      44             :         /* nodes for data structures */
      45             :         tommy_node node;
      46             : };
      47             : 
      48             : /**
      49             :  * Removes the specified link from the data set.
      50             :  */
      51         956 : static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* link)
      52             : {
      53         956 :         struct snapraid_state* state = scan->state;
      54         956 :         struct snapraid_disk* disk = scan->disk;
      55             : 
      56             :         /* state changed */
      57         956 :         state->need_write = 1;
      58             : 
      59             :         /* remove the file from the link containers */
      60         956 :         tommy_hashdyn_remove_existing(&disk->linkset, &link->nodeset);
      61         956 :         tommy_list_remove_existing(&disk->linklist, &link->nodelist);
      62             : 
      63             :         /* deallocate */
      64         956 :         link_free(link);
      65         956 : }
      66             : 
      67             : /**
      68             :  * Inserts the specified link in the data set.
      69             :  */
      70        2457 : static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* link)
      71             : {
      72        2457 :         struct snapraid_state* state = scan->state;
      73        2457 :         struct snapraid_disk* disk = scan->disk;
      74             : 
      75             :         /* state changed */
      76        2457 :         state->need_write = 1;
      77             : 
      78             :         /* insert the link in the link containers */
      79        2457 :         tommy_hashdyn_insert(&disk->linkset, &link->nodeset, link, link_name_hash(link->sub));
      80        2457 :         tommy_list_insert_tail(&disk->linklist, &link->nodelist, link);
      81        2457 : }
      82             : 
      83             : /**
      84             :  * Processes a symbolic link.
      85             :  */
      86       31866 : static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub, const char* linkto, unsigned link_flag)
      87             : {
      88       31866 :         struct snapraid_state* state = scan->state;
      89       31866 :         struct snapraid_disk* disk = scan->disk;
      90             :         struct snapraid_link* link;
      91             : 
      92             :         /* check if the link already exists */
      93       31866 :         link = tommy_hashdyn_search(&disk->linkset, link_name_compare_to_arg, sub, link_name_hash(sub));
      94       31866 :         if (link) {
      95             :                 /* check if multiple files have the same name */
      96       29409 :                 if (link_flag_has(link, FILE_IS_PRESENT)) {
      97             :                         /* LCOV_EXCL_START */
      98             :                         log_fatal("Internal inconsistency for link '%s%s'\n", disk->dir, sub);
      99             :                         exit(EXIT_FAILURE);
     100             :                         /* LCOV_EXCL_STOP */
     101             :                 }
     102             : 
     103             :                 /* mark as present */
     104       29409 :                 link_flag_set(link, FILE_IS_PRESENT);
     105             : 
     106             :                 /* check if the link is not changed and it's of the same kind */
     107       29409 :                 if (strcmp(link->linkto, linkto) == 0 && link_flag == link_flag_get(link, FILE_IS_LINK_MASK)) {
     108             :                         /* it's equal */
     109       29409 :                         ++scan->count_equal;
     110             : 
     111       58818 :                         if (state->opt.gui) {
     112         829 :                                 log_tag("scan:equal:%s:%s\n", disk->name, esc(link->sub));
     113             :                         }
     114             :                 } else {
     115             :                         /* it's an update */
     116             : 
     117             :                         /* we have to save the linkto/type */
     118           0 :                         state->need_write = 1;
     119             : 
     120           0 :                         ++scan->count_change;
     121             : 
     122           0 :                         log_tag("scan:update:%s:%s\n", disk->name, esc(link->sub));
     123           0 :                         if (is_diff) {
     124           0 :                                 printf("update %s%s\n", disk->dir, link->sub);
     125             :                         }
     126             : 
     127             :                         /* update it */
     128           0 :                         free(link->linkto);
     129           0 :                         link->linkto = strdup_nofail(linkto);
     130           0 :                         link_flag_let(link, link_flag, FILE_IS_LINK_MASK);
     131             :                 }
     132             : 
     133             :                 /* nothing more to do */
     134       61275 :                 return;
     135             :         } else {
     136             :                 /* create the new link */
     137        2457 :                 ++scan->count_insert;
     138             : 
     139        2457 :                 log_tag("scan:add:%s:%s\n", disk->name, esc(sub));
     140        2457 :                 if (is_diff) {
     141         598 :                         printf("add %s%s\n", disk->dir, sub);
     142             :                 }
     143             : 
     144             :                 /* and continue to insert it */
     145             :         }
     146             : 
     147             :         /* insert it */
     148        2457 :         link = link_alloc(sub, linkto, link_flag);
     149             : 
     150             :         /* mark it as present */
     151        2457 :         link_flag_set(link, FILE_IS_PRESENT);
     152             : 
     153             :         /* insert it in the delayed insert list */
     154        2457 :         tommy_list_insert_tail(&scan->link_insert_list, &link->nodelist, link);
     155             : }
     156             : 
     157             : /**
     158             :  * Removes the specified file from the parity.
     159             :  */
     160       55207 : static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
     161             : {
     162       55207 :         struct snapraid_state* state = scan->state;
     163       55207 :         struct snapraid_disk* disk = scan->disk;
     164             : 
     165             :         block_off_t i;
     166             : 
     167             :         /* state changed */
     168       55207 :         state->need_write = 1;
     169             : 
     170             :         /* free all the blocks of the file */
     171      191141 :         for (i = 0; i < file->blockmax; ++i) {
     172      135934 :                 struct snapraid_block* block = &file->blockvec[i];
     173      135934 :                 block_off_t block_pos = block->parity_pos;
     174             :                 unsigned block_state;
     175             :                 struct snapraid_deleted* deleted;
     176             : 
     177             :                 /* adjust the first free position */
     178             :                 /* note that doing all the deletions before alllocations, */
     179             :                 /* first_free_block is always 0 and the "if" is never triggered */
     180             :                 /* but we keep this code anyway for completeness. */
     181      135934 :                 if (disk->first_free_block > block_pos)
     182           0 :                         disk->first_free_block = block_pos;
     183             : 
     184             :                 /* allocated a new deleted block from the block we are going to delete */
     185      135934 :                 deleted = deleted_dup(block);
     186             : 
     187             :                 /* in case we scan after an aborted sync, */
     188             :                 /* we could get also intermediate states */
     189      135934 :                 block_state = block_state_get(block);
     190      135934 :                 switch (block_state) {
     191             :                 case BLOCK_STATE_BLK :
     192             :                         /* we keep the hash making it an "old" hash, because the parity is still containing data for it */
     193       31139 :                         break;
     194             :                 case BLOCK_STATE_CHG :
     195             :                         /* if we have not already cleared the past hash */
     196       57633 :                         if (!state->clear_past_hash) {
     197             :                                 /* in these cases we don't know if the old state is still the one */
     198             :                                 /* stored inside the parity, because after an aborted sync, the parity */
     199             :                                 /* may be or may be not have been updated with the data that it's now */
     200             :                                 /* deleted. Then we reset the hash to a bogus value. */
     201             :                                 /* For example: */
     202             :                                 /* - One file is added */
     203             :                                 /* - Sync aborted after updating the parity to the new state, */
     204             :                                 /*   but without saving the content file representing this new state. */
     205             :                                 /* - File is now deleted after the aborted sync */
     206             :                                 /* - Sync again, deleting the blocks (exactly here) */
     207             :                                 /*   with the hash of CHG block not represeting the real parity state */
     208           0 :                                 hash_invalid_set(deleted->block.hash);
     209             :                         }
     210       57633 :                         break;
     211             :                 case BLOCK_STATE_REP :
     212             :                         /* we just don't know the old hash, and then we set it to invalid */
     213       47162 :                         hash_invalid_set(deleted->block.hash);
     214       47162 :                         break;
     215             :                 default :
     216             :                         /* LCOV_EXCL_START */
     217             :                         log_fatal("Internal inconsistency in deallocating for block %u state %u\n", block->parity_pos, block_state);
     218             :                         exit(EXIT_FAILURE);
     219             :                         /* LCOV_EXCL_STOP */
     220             :                 }
     221             : 
     222             :                 /* insert it in the list of deleted blocks */
     223      135934 :                 tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
     224             : 
     225             :                 /* set the deleted block in the block array */
     226      135934 :                 tommy_arrayblk_set(&disk->blockarr, block_pos, &deleted->block);
     227             :         }
     228       55207 : }
     229             : 
     230             : /**
     231             :  * Checks if a file is completely formed of blocks with invalid parity,
     232             :  * and no rehash is tagged, and if it has at least one block.
     233             :  */
     234      897263 : static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_file* file)
     235             : {
     236             :         block_off_t i;
     237             : 
     238             :         /* with no block, it never has an invalid parity */
     239      897263 :         if (file->blockmax == 0)
     240        1187 :                 return 0;
     241             : 
     242             :         /* check all blocks */
     243      951882 :         for (i = 0; i < file->blockmax; ++i) {
     244      929217 :                 struct snapraid_block* block = &file->blockvec[i];
     245             :                 snapraid_info info;
     246             : 
     247             :                 /* exclude blocks with parity */
     248      929217 :                 if (!block_has_invalid_parity(block))
     249      873411 :                         return 0;
     250             : 
     251             :                 /* get block specific info */
     252       55806 :                 info = info_get(&state->infoarr, block->parity_pos);
     253             : 
     254             :                 /* if rehash fails */
     255       55806 :                 if (info_get_rehash(info))
     256           0 :                         return 0;
     257             :         }
     258             : 
     259       22665 :         return 1;
     260             : }
     261             : 
     262             : /**
     263             :  * Checks if a file is completely formed of blocks with an updated hash,
     264             :  * and no rehash is tagged, and if it has at least one block.
     265             :  */
     266       23658 : static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_file* file)
     267             : {
     268             :         block_off_t i;
     269             : 
     270             :         /* with no block, it never has a hash */
     271       23658 :         if (file->blockmax == 0)
     272          27 :                 return 0;
     273             : 
     274             :         /* check all blocks */
     275       42139 :         for (i = 0; i < file->blockmax; ++i) {
     276       35994 :                 struct snapraid_block* block = &file->blockvec[i];
     277             :                 snapraid_info info;
     278             : 
     279             :                 /* exclude blocks without hash */
     280       35994 :                 if (!block_has_updated_hash(block))
     281        1702 :                         return 0;
     282             : 
     283             :                 /* get block specific info */
     284       34292 :                 info = info_get(&state->infoarr, block->parity_pos);
     285             : 
     286             :                 /* exclude blocks needing a rehash */
     287       34292 :                 if (info_get_rehash(info))
     288       15784 :                         return 0;
     289             :         }
     290             : 
     291        6145 :         return 1;
     292             : }
     293             : 
     294      100042 : static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid_file* file)
     295             : {
     296      100042 :         struct snapraid_state* state = scan->state;
     297      100042 :         struct snapraid_disk* disk = scan->disk;
     298             : 
     299             :         /* if we sort for physical offsets we have to read them for new files */
     300      100042 :         if (state->opt.force_order == SORT_PHYSICAL
     301       11430 :                 && file->physical == FILEPHY_UNREAD_OFFSET
     302             :         ) {
     303             :                 char path_next[PATH_MAX];
     304             : 
     305       11430 :                 pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, file->sub);
     306             : 
     307       11430 :                 if (filephy(path_next, file->size, &file->physical) != 0) {
     308             :                         /* LCOV_EXCL_START */
     309             :                         log_fatal("Error in getting the physical offset of file '%s'. %s.\n", path_next, strerror(errno));
     310             :                         exit(EXIT_FAILURE);
     311             :                         /* LCOV_EXCL_STOP */
     312             :                 }
     313             :         }
     314             : 
     315      100042 :         tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
     316      100042 : }
     317             : 
     318             : /**
     319             :  * Refresh the file info.
     320             :  *
     321             :  * This is needed by Windows as the normal way to list directories may report not
     322             :  * updated info. Only the GetFileInformationByHandle() func, called file-by-file,
     323             :  * really ensures to return synced info.
     324             :  *
     325             :  * If this happens, we read also the physical offset, to avoid to read it later.
     326             :  */
     327       77377 : static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struct stat* st, uint64_t* physical)
     328             : {
     329             : #if HAVE_LSTAT_SYNC
     330             :         struct snapraid_state* state = scan->state;
     331             :         struct snapraid_disk* disk = scan->disk;
     332             : 
     333             :         /* if the st_sync is not set, ensure to get synced info */
     334             :         if (st->st_sync == 0) {
     335             :                 char path_next[PATH_MAX];
     336             :                 struct stat synced_st;
     337             : 
     338             :                 pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, sub);
     339             : 
     340             :                 /* if we sort for physical offsets we have to read them for new files */
     341             :                 if (state->opt.force_order == SORT_PHYSICAL
     342             :                         && *physical == FILEPHY_UNREAD_OFFSET
     343             :                 ) {
     344             :                         /* do nothing, leave the pointer to read the physical offset */
     345             :                 } else {
     346             :                         physical = 0; /* set the pointer to 0 to read nothing */
     347             :                 }
     348             : 
     349             :                 if (lstat_sync(path_next, &synced_st, physical) != 0) {
     350             :                         /* LCOV_EXCL_START */
     351             :                         log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
     352             :                         exit(EXIT_FAILURE);
     353             :                         /* LCOV_EXCL_STOP */
     354             :                 }
     355             : 
     356             :                 if (st->st_mtime != synced_st.st_mtime
     357             :                         || st->st_mtimensec != synced_st.st_mtimensec
     358             :                 ) {
     359             : #ifndef _WIN32
     360             :                         /*
     361             :                          * In Windows having different metadata is expected with open files
     362             :                          * because the metadata in the directory is updated only when the file
     363             :                          * is closed.
     364             :                          *
     365             :                          * See also:
     366             :                          * Why is the file size reported incorrectly for files that are still being written to?
     367             :                          * http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx
     368             :                          */
     369             :                         log_fatal("WARNING! Detected uncached time change for file '%s'\n", sub);
     370             :                         log_fatal("It's better if you run SnapRAID without other processes running.\n");
     371             : #endif
     372             :                         st->st_mtime = synced_st.st_mtime;
     373             :                         st->st_mtimensec = synced_st.st_mtimensec;
     374             :                 }
     375             : 
     376             :                 if (st->st_size != synced_st.st_size) {
     377             : #ifndef _WIN32
     378             :                         log_fatal("WARNING! Detected uncached size change for file '%s'\n", sub);
     379             :                         log_fatal("It's better if you run SnapRAID without other processes running.\n");
     380             : #endif
     381             :                         st->st_size = synced_st.st_size;
     382             :                 }
     383             : 
     384             :                 if (st->st_ino != synced_st.st_ino) {
     385             :                         log_fatal("DANGER! Detected uncached inode change for file '%s'\n", sub);
     386             :                         log_fatal("Please ensure to run SnapRAID without other processes running.\n");
     387             :                         /* at this point, it's too late to change inode */
     388             :                         /* and having inconsistent inodes may result to internal failures */
     389             :                         /* so, it's better to abort */
     390             :                         exit(EXIT_FAILURE);
     391             :                 }
     392             :         }
     393             : #else
     394             :         (void)scan;
     395             :         (void)sub;
     396             :         (void)st;
     397             :         (void)physical;
     398             : #endif
     399       77377 : }
     400             : 
     401             : /**
     402             :  * Keeps the file as it's (or with only a name/inode modification).
     403             :  *
     404             :  * If the file is kept, nothing has to be done.
     405             :  *
     406             :  * But if a file contains only blocks with invalid parity, it's reallocated to ensure
     407             :  * to always minimize the space used in the parity.
     408             :  *
     409             :  * This could happen after a failed sync, when some other files are deleted,
     410             :  * and then new ones can be moved backward to fill the hole created.
     411             :  */
     412      897263 : static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* file)
     413             : {
     414      897263 :         struct snapraid_disk* disk = scan->disk;
     415             : 
     416             :         /* if the file is full invalid, schedule a reinsert at later stage */
     417      897263 :         if (file_is_full_invalid_parity_and_stable(scan->state, file)) {
     418             :                 /* deallocate the file from the parity */
     419       22665 :                 scan_file_deallocate(scan, file);
     420             : 
     421             :                 /* remove the file from the list */
     422       22665 :                 tommy_list_remove_existing(&disk->filelist, &file->nodelist);
     423             : 
     424             :                 /* insert the file in the delayed block allocation */
     425       22665 :                 scan_file_delayed_insert(scan, file);
     426             :         }
     427      897263 : }
     428             : 
     429             : /**
     430             :  * Removes the specified file from the data set.
     431             :  */
     432       32542 : static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
     433             : {
     434       32542 :         struct snapraid_disk* disk = scan->disk;
     435             : 
     436             :         /* deallocate the file from the parity */
     437       32542 :         scan_file_deallocate(scan, file);
     438             : 
     439             :         /* remove the file from the file containers */
     440       32542 :         if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
     441       32501 :                 tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
     442             :         }
     443       32542 :         tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
     444       32542 :         tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
     445       32542 :         tommy_list_remove_existing(&disk->filelist, &file->nodelist);
     446             : 
     447             :         /* deallocate */
     448       32542 :         file_free(file);
     449       32542 : }
     450             : 
     451             : /**
     452             :  * Inserts the specified file in the data set.
     453             :  */
     454      100042 : static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
     455             : {
     456      100042 :         struct snapraid_state* state = scan->state;
     457      100042 :         struct snapraid_disk* disk = scan->disk;
     458             :         block_off_t i;
     459             :         block_off_t block_max;
     460             :         block_off_t block_pos;
     461             : 
     462             :         /* state changed */
     463      100042 :         state->need_write = 1;
     464             : 
     465             :         /* allocate the blocks of the file */
     466      100042 :         block_pos = disk->first_free_block;
     467      100042 :         block_max = tommy_arrayblk_size(&disk->blockarr);
     468      346361 :         for (i = 0; i < file->blockmax; ++i) {
     469             :                 snapraid_info info;
     470      246319 :                 struct snapraid_block* block = &file->blockvec[i];
     471             : 
     472             :                 /* increment the position until the first really free block */
     473      994796 :                 while (block_pos < block_max && block_has_file(tommy_arrayblk_get(&disk->blockarr, block_pos)))
     474      502158 :                         ++block_pos;
     475             : 
     476             :                 /* if not found, allocate a new one */
     477      246319 :                 if (block_pos == block_max) {
     478      155209 :                         ++block_max;
     479      155209 :                         tommy_arrayblk_grow(&disk->blockarr, block_max);
     480             :                 }
     481             : 
     482             :                 /* set the position */
     483      246319 :                 block->parity_pos = block_pos;
     484             : 
     485             :                 /* get block specific info */
     486      246319 :                 info = info_get(&state->infoarr, block_pos);
     487             : 
     488             :                 /* if the file block already has an updated hash without rehash */
     489      246319 :                 if (block_has_updated_hash(block) && !info_get_rehash(info)) {
     490             :                         /* the only possible case is for REP blocks */
     491       39390 :                         assert(block_state_get(block) == BLOCK_STATE_REP);
     492             : 
     493             :                         /* convert to a REP block */
     494       39390 :                         block_state_set(&file->blockvec[i], BLOCK_STATE_REP);
     495             : 
     496             :                         /* and keep the hash as it's */
     497             :                 } else {
     498             :                         struct snapraid_block* over_block;
     499             :                         unsigned block_state;
     500             : 
     501             :                         /* convert to a CHG block */
     502      206929 :                         block_state_set(block, BLOCK_STATE_CHG);
     503             : 
     504             :                         /* block to overwrite */
     505      206929 :                         over_block = tommy_arrayblk_get(&disk->blockarr, block_pos);
     506             : 
     507             :                         /* state of the block we are going to overwrite */
     508      206929 :                         block_state = block_state_get(over_block);
     509             : 
     510             :                         /* if the block is an empty one */
     511      206929 :                         if (block_state == BLOCK_STATE_EMPTY) {
     512             :                                 /* the block was empty and filled with zeros */
     513             :                                 /* set the hash to the special ZERO value */
     514      153240 :                                 hash_zero_set(block->hash);
     515             :                         } else {
     516             :                                 /* otherwise it's a DELETED one */
     517       53689 :                                 assert(block_state == BLOCK_STATE_DELETED);
     518             : 
     519             :                                 /* copy the past hash of the block */
     520       53689 :                                 memcpy(block->hash, over_block->hash, HASH_SIZE);
     521             : 
     522             :                                 /* if we have not already cleared the past hash */
     523       53689 :                                 if (!state->clear_past_hash) {
     524             :                                         /* in this case we don't know if the old state is still the one */
     525             :                                         /* stored inside the parity, because after an aborted sync, the parity */
     526             :                                         /* may be or may be not have been updated with the new data */
     527             :                                         /* Then we reset the hash to a bogus value */
     528             :                                         /* For example: */
     529             :                                         /* - One file is deleted */
     530             :                                         /* - Sync aborted after, updating the parity to the new state, */
     531             :                                         /*   but without saving the content file representing this new state. */
     532             :                                         /* - Another file is added again (exactly here) */
     533             :                                         /*   with the hash of DELETED block not represeting the real parity state */
     534         380 :                                         hash_invalid_set(block->hash);
     535             :                                 }
     536             :                         }
     537             :                 }
     538             : 
     539             :                 /* store in the disk map, after invalidating all the other blocks */
     540      246319 :                 tommy_arrayblk_set(&disk->blockarr, block_pos, block);
     541             : 
     542             :                 /* set the new free position */
     543      246319 :                 disk->first_free_block = block_pos + 1;
     544             :         }
     545             : 
     546             :         /* note that the file is already added in the file hashtables */
     547      100042 :         tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
     548      100042 : }
     549             : 
     550             : /**
     551             :  * Processes a file.
     552             :  */
     553      974966 : static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, struct stat* st, uint64_t physical)
     554             : {
     555      974966 :         struct snapraid_state* state = scan->state;
     556      974966 :         struct snapraid_disk* disk = scan->disk;
     557             :         struct snapraid_file* file;
     558             :         tommy_node* i;
     559             :         int is_original_file_size_different_than_zero;
     560             :         int is_file_already_present;
     561             :         int is_file_reported;
     562             : 
     563             :         /*
     564             :          * If the disk has persistent inodes and UUID, try a search on the past inodes,
     565             :          * to detect moved files.
     566             :          *
     567             :          * For persistent inodes we mean inodes that keep their values when the filesystem
     568             :          * is unmounted and remounted. This don't always happen.
     569             :          *
     570             :          * Cases found are:
     571             :          * - Linux FUSE with exFAT driver from https://code.google.com/p/exfat/.
     572             :          *   Inodes are reassigned at every mount restarting from 1 and incrementing.
     573             :          *   As worse, the exFAT support in FUSE doesn't use subsecond precision in timestamps
     574             :          *   making inode collision more easy (exFAT by design supports 10ms precision).
     575             :          * - Linux VFAT kernel (3.2) driver. Inodes are fully reassigned at every mount.
     576             :          *
     577             :          * In such cases, to avoid possible random collisions, it's better to disable the moved
     578             :          * file recognition.
     579             :          *
     580             :          * For persistent UUID we mean that it has the same UUID as before.
     581             :          * Otherwise, if the UUID is changed, likely it's a new recreated filesystem,
     582             :          * and then the inode have no meaning.
     583             :          *
     584             :          * Note that to disable the search by past inode, we do this implicitely
     585             :          * removing all the past inode before searching for files.
     586             :          * This ensures that no file is found with a past inode, but at the same time,
     587             :          * it allows to find new files with the same inode, to identify them as hardlinks.
     588             :          */
     589      974966 :         int has_past_inodes = !disk->has_volatile_inodes && !disk->has_different_uuid && !disk->has_unsupported_uuid;
     590             : 
     591             :         /* always search with the new inode, in the all new inodes found until now, */
     592             :         /* with the eventual presence of also the past inodes */
     593      974966 :         uint64_t inode = st->st_ino;
     594             : 
     595      974966 :         file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
     596             : 
     597             :         /* identify moved files with past inodes and hardlinks with the new inodes */
     598      974966 :         if (file) {
     599             :                 /* check if the file is not changed */
     600      811432 :                 if (file->size == st->st_size
     601      803087 :                         && file->mtime_sec == st->st_mtime
     602      801536 :                         && (file->mtime_nsec == STAT_NSEC(st)
     603             :                         /* always accept the stored value if it's STAT_NSEC_INVALID */
     604             :                         /* it happens when upgrading from an old version of SnapRAID */
     605             :                         /* not yet supporting the nanosecond field */
     606           0 :                         || file->mtime_nsec == STAT_NSEC_INVALID
     607             :                         )
     608             :                 ) {
     609             :                         /* check if multiple files have the same inode */
     610      801536 :                         if (file_flag_has(file, FILE_IS_PRESENT)) {
     611             : #if HAVE_STRUCT_STAT_ST_NLINK
     612         326 :                                 if (st->st_nlink <= 1) {
     613             :                                         /* LCOV_EXCL_START */
     614             :                                         log_fatal("Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub);
     615             :                                         exit(EXIT_FAILURE);
     616             :                                         /* LCOV_EXCL_STOP */
     617             :                                 }
     618             : #endif
     619             :                                 /* it's a hardlink */
     620         326 :                                 scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
     621         326 :                                 return;
     622             :                         }
     623             : 
     624             :                         /* mark as present */
     625      801210 :                         file_flag_set(file, FILE_IS_PRESENT);
     626             : 
     627             :                         /* update the nanoseconds mtime only if different */
     628             :                         /* to avoid unneeded updates */
     629      801210 :                         if (file->mtime_nsec == STAT_NSEC_INVALID
     630           0 :                                 && STAT_NSEC(st) != file->mtime_nsec
     631             :                         ) {
     632           0 :                                 file->mtime_nsec = STAT_NSEC(st);
     633             : 
     634             :                                 /* we have to save the new mtime */
     635           0 :                                 state->need_write = 1;
     636             :                         }
     637             : 
     638      801210 :                         if (strcmp(file->sub, sub) != 0) {
     639             :                                 /* if the path is different, it means a moved file with the same inode */
     640         155 :                                 ++scan->count_move;
     641             : 
     642         155 :                                 log_tag("scan:move:%s:%s:%s\n", disk->name, esc(file->sub), esc(sub));
     643         155 :                                 if (is_diff) {
     644           0 :                                         printf("move %s%s -> %s%s\n", disk->dir, file->sub, disk->dir, sub);
     645             :                                 }
     646             : 
     647             :                                 /* remove from the name set */
     648         155 :                                 tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
     649             : 
     650             :                                 /* save the new name */
     651         155 :                                 file_rename(file, sub);
     652             : 
     653             :                                 /* reinsert in the name set */
     654         155 :                                 tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
     655             : 
     656             :                                 /* we have to save the new name */
     657         155 :                                 state->need_write = 1;
     658             :                         } else {
     659             :                                 /* otherwise it's equal */
     660      801055 :                                 ++scan->count_equal;
     661             : 
     662      801055 :                                 if (state->opt.gui) {
     663       22555 :                                         log_tag("scan:equal:%s:%s\n", disk->name, esc(file->sub));
     664             :                                 }
     665             :                         }
     666             : 
     667             :                         /* mark the file as kept */
     668      801210 :                         scan_file_keep(scan, file);
     669             : 
     670             :                         /* nothing more to do */
     671      801210 :                         return;
     672             :                 }
     673             : 
     674             :                 /* here the file matches the inode, but not the other info */
     675             :                 /* if could be a modified file with the same name, */
     676             :                 /* or a restored/copied file that get assigned a previously used inode, */
     677             :                 /* or a filesystem with not persistent inodes */
     678             : 
     679             :                 /* for sure it cannot be already present */
     680        9896 :                 if (file_flag_has(file, FILE_IS_PRESENT)) {
     681             :                         /* LCOV_EXCL_START */
     682             :                         log_fatal("Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' matching and already present but different\n", file->inode, disk->dir, sub, disk->dir, file->sub);
     683             :                         exit(EXIT_FAILURE);
     684             :                         /* LCOV_EXCL_STOP */
     685             :                 }
     686             : 
     687             :                 /* assume a previously used inode, it's the worst case */
     688             :                 /* and we handle it removing the duplicate stored inode. */
     689             :                 /* If the file is found by name (not necessarely in this function call), */
     690             :                 /* it will have the inode restored, otherwise, it will get removed */
     691             : 
     692             :                 /* remove from the inode set */
     693        9896 :                 tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
     694             : 
     695             :                 /* clear the inode */
     696             :                 /* this is not really needed for correct functionality */
     697             :                 /* because we are going to set FILE_IS_WITHOUT_INODE */
     698             :                 /* but it's easier for debugging to have invalid inodes set to 0 */
     699        9896 :                 file->inode = 0;
     700             : 
     701             :                 /* mark as missing inode */
     702        9896 :                 file_flag_set(file, FILE_IS_WITHOUT_INODE);
     703             : 
     704             :                 /* go further to find it by name */
     705             :         }
     706             : 
     707             :         /* initialize for later overwrite */
     708      173430 :         is_file_reported = 0;
     709      173430 :         is_original_file_size_different_than_zero = 0;
     710             : 
     711             :         /* then try finding it by name */
     712      173430 :         file = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub, file_path_hash(sub));
     713             : 
     714             :         /* keep track if the file already exists */
     715      173430 :         is_file_already_present = file != 0;
     716             : 
     717      173430 :         if (is_file_already_present) {
     718             :                 /* if the file is without an inode */
     719       98966 :                 if (file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
     720             :                         /* set it now */
     721        9855 :                         file->inode = st->st_ino;
     722             : 
     723             :                         /* insert in the set */
     724        9855 :                         tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
     725             : 
     726             :                         /* unmark as missing inode */
     727        9855 :                         file_flag_clear(file, FILE_IS_WITHOUT_INODE);
     728             :                 } else {
     729             :                         /* here the inode has to be different, otherwise we would have found it before */
     730       89111 :                         if (file->inode == st->st_ino) {
     731             :                                 /* LCOV_EXCL_START */
     732             :                                 log_fatal("Internal inconsistency in inode '%" PRIu64 "' for files '%s%s' as unexpected matching\n", file->inode, disk->dir, sub);
     733             :                                 exit(EXIT_FAILURE);
     734             :                                 /* LCOV_EXCL_STOP */
     735             :                         }
     736             :                 }
     737             : 
     738             :                 /* for sure it cannot be already present */
     739       98966 :                 if (file_flag_has(file, FILE_IS_PRESENT)) {
     740             :                         /* LCOV_EXCL_START */
     741             :                         log_fatal("Internal inconsistency in path for file '%s%s' matching and already present\n", disk->dir, sub);
     742             :                         exit(EXIT_FAILURE);
     743             :                         /* LCOV_EXCL_STOP */
     744             :                 }
     745             : 
     746             :                 /* check if the file is not changed */
     747       98966 :                 if (file->size == st->st_size
     748       98372 :                         && file->mtime_sec == st->st_mtime
     749       96053 :                         && (file->mtime_nsec == STAT_NSEC(st)
     750             :                         /* always accept the stored value if it's STAT_NSEC_INVALID */
     751             :                         /* it happens when upgrading from an old version of SnapRAID */
     752             :                         /* not yet supporting the nanosecond field */
     753           0 :                         || file->mtime_nsec == STAT_NSEC_INVALID
     754             :                         )
     755             :                 ) {
     756             :                         /* mark as present */
     757       96053 :                         file_flag_set(file, FILE_IS_PRESENT);
     758             : 
     759             :                         /* update the nano seconds mtime only if different */
     760             :                         /* to avoid unneeded updates */
     761       96053 :                         if (file->mtime_nsec == STAT_NSEC_INVALID
     762           0 :                                 && STAT_NSEC(st) != STAT_NSEC_INVALID
     763             :                         ) {
     764           0 :                                 file->mtime_nsec = STAT_NSEC(st);
     765             : 
     766             :                                 /* we have to save the new mtime */
     767           0 :                                 state->need_write = 1;
     768             :                         }
     769             : 
     770             :                         /* if when processing the disk we used the past inodes values */
     771       96053 :                         if (has_past_inodes) {
     772             :                                 /* if persistent inodes are supported, we are sure that the inode number */
     773             :                                 /* is now different, because otherwise the file would have been found */
     774             :                                 /* when searching by inode. */
     775             :                                 /* if the inode is different, it means a rewritten file with the same path */
     776             :                                 /* like when restoring a backup that restores also the timestamp */
     777       96053 :                                 ++scan->count_restore;
     778             : 
     779       96053 :                                 log_tag("scan:restore:%s:%s\n", disk->name, esc(sub));
     780       96053 :                                 if (is_diff) {
     781           0 :                                         printf("restore %s%s\n", disk->dir, sub);
     782             :                                 }
     783             : 
     784             :                                 /* remove from the inode set */
     785       96053 :                                 tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
     786             : 
     787             :                                 /* save the new inode */
     788       96053 :                                 file->inode = st->st_ino;
     789             : 
     790             :                                 /* reinsert in the inode set */
     791       96053 :                                 tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
     792             : 
     793             :                                 /* we have to save the new inode */
     794       96053 :                                 state->need_write = 1;
     795             :                         } else {
     796             :                                 /* otherwise it's the case of not persistent inode, where doesn't */
     797             :                                 /* matter if the inode is different or equal, because they have no */
     798             :                                 /* meaning, and then we don't even save them */
     799           0 :                                 ++scan->count_equal;
     800             : 
     801           0 :                                 if (state->opt.gui) {
     802           0 :                                         log_tag("scan:equal:%s:%s\n", disk->name, esc(file->sub));
     803             :                                 }
     804             :                         }
     805             : 
     806             :                         /* mark the file as kept */
     807       96053 :                         scan_file_keep(scan, file);
     808             : 
     809             :                         /* nothing more to do */
     810       96053 :                         return;
     811             :                 }
     812             : 
     813             :                 /* here if the file is changed but with the correct name */
     814             : 
     815             :                 /* keep track if the original file was not of zero size */
     816        2913 :                 is_original_file_size_different_than_zero = file->size != 0;
     817             : 
     818             :                 /* remove it, and continue to insert it again */
     819        2913 :                 scan_file_remove(scan, file);
     820             : 
     821             :                 /* and continue to insert it again */
     822             :         }
     823             : 
     824             :         /* refresh the info, to ensure that they are synced, */
     825             :         /* note that we refresh only the info of the new or modified files */
     826             :         /* because this is slow operation */
     827       77377 :         scan_file_refresh(scan, sub, st, &physical);
     828             : 
     829             :         /* do a safety check to ensure that the common ext4 case of zeroing */
     830             :         /* the size of a file after a crash doesn't propagate to the backup */
     831       77377 :         if (is_original_file_size_different_than_zero && st->st_size == 0) {
     832           0 :                 if (!state->opt.force_zero) {
     833             :                         /* LCOV_EXCL_START */
     834             :                         log_fatal("The file '%s%s' has unexpected zero size!\n", disk->dir, sub);
     835             :                         log_fatal("It's possible that after a kernel crash this file was lost,\n");
     836             :                         log_fatal("and you can use 'snapraid --filter %s fix' to recover it.\n", sub);
     837             :                         if (!is_diff) {
     838             :                                 log_fatal("If this an expected condition you can '%s' anyway using 'snapraid --force-zero %s'\n", state->command, state->command);
     839             :                                 exit(EXIT_FAILURE);
     840             :                         }
     841             :                         /* LCOV_EXCL_STOP */
     842             :                 }
     843             :         }
     844             : 
     845             :         /* insert it */
     846       77377 :         file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, STAT_NSEC(st), st->st_ino, physical);
     847             : 
     848             :         /* mark it as present */
     849       77377 :         file_flag_set(file, FILE_IS_PRESENT);
     850             : 
     851             :         /* if copy detection is enabled */
     852             :         /* note that the copy detection is tried also for updated files */
     853             :         /* this makes sense because it may happen to have two different copies */
     854             :         /* of the same file, and we move the right one over the wrong one */
     855             :         /* in such case we have a "copy" over an "update" */
     856       77377 :         if (!state->opt.force_nocopy) {
     857       77377 :                 tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
     858             : 
     859             :                 /* search for a file with the same name and stamp in all the disks */
     860      506296 :                 for (i = state->disklist; i != 0; i = i->next) {
     861      435064 :                         struct snapraid_disk* other_disk = i->data;
     862             :                         struct snapraid_file* other_file;
     863             : 
     864             :                         /* if the nanosecond part of the time stamp is valid, search */
     865             :                         /* for name and stamp, otherwise for path and stamp */
     866      435064 :                         if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
     867      434956 :                                 other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
     868             :                         else
     869         108 :                                 other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
     870             : 
     871             :                         /* if found, and it's a fully hashed file */
     872      435064 :                         if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) {
     873             :                                 /* assume that the file is a copy, and reuse the hash */
     874        6145 :                                 file_copy(other_file, file);
     875             : 
     876             :                                 /* revert old counter and use the copy one */
     877        6145 :                                 ++scan->count_copy;
     878             : 
     879        6145 :                                 log_tag("scan:copy:%s:%s:%s:%s\n", other_disk->name, esc(other_file->sub), disk->name, esc(file->sub));
     880        6145 :                                 if (is_diff) {
     881          31 :                                         printf("copy %s%s -> %s%s\n", other_disk->dir, other_file->sub, disk->dir, file->sub);
     882             :                                 }
     883             : 
     884             :                                 /* mark it as reported */
     885        6145 :                                 is_file_reported = 1;
     886             : 
     887             :                                 /* no need to continue the search */
     888        6145 :                                 break;
     889             :                         }
     890             :                 }
     891             :         }
     892             : 
     893             :         /* if not yet reported, do it now */
     894       77377 :         if (!is_file_reported) {
     895       71232 :                 if (is_file_already_present) {
     896        2913 :                         ++scan->count_change;
     897             : 
     898        2913 :                         log_tag("scan:update:%s:%s\n", disk->name, esc(sub));
     899        2913 :                         if (is_diff) {
     900         117 :                                 printf("update %s%s\n", disk->dir, sub);
     901             :                         }
     902             :                 } else {
     903       68319 :                         ++scan->count_insert;
     904             : 
     905       68319 :                         log_tag("scan:add:%s:%s\n", disk->name, esc(sub));
     906       68319 :                         if (is_diff) {
     907       17043 :                                 printf("add %s%s\n", disk->dir, sub);
     908             :                         }
     909             :                 }
     910             :         }
     911             : 
     912             :         /* insert the file in the file hashtables, to allow to find duplicate hardlinks */
     913       77377 :         tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
     914       77377 :         tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
     915       77377 :         tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
     916             : 
     917             :         /* insert the file in the delayed block allocation */
     918       77377 :         scan_file_delayed_insert(scan, file);
     919             : }
     920             : 
     921             : /**
     922             :  * Removes the specified dir from the data set.
     923             :  */
     924           2 : static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
     925             : {
     926           2 :         struct snapraid_state* state = scan->state;
     927           2 :         struct snapraid_disk* disk = scan->disk;
     928             : 
     929             :         /* state changed */
     930           2 :         state->need_write = 1;
     931             : 
     932             :         /* remove the file from the dir containers */
     933           2 :         tommy_hashdyn_remove_existing(&disk->dirset, &dir->nodeset);
     934           2 :         tommy_list_remove_existing(&disk->dirlist, &dir->nodelist);
     935             : 
     936             :         /* deallocate */
     937           2 :         dir_free(dir);
     938           2 : }
     939             : 
     940             : /**
     941             :  * Inserts the specified dir in the data set.
     942             :  */
     943          10 : static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
     944             : {
     945          10 :         struct snapraid_state* state = scan->state;
     946          10 :         struct snapraid_disk* disk = scan->disk;
     947             : 
     948             :         /* state changed */
     949          10 :         state->need_write = 1;
     950             : 
     951             :         /* insert the dir in the dir containers */
     952          10 :         tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
     953          10 :         tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
     954          10 : }
     955             : 
     956             : /**
     957             :  * Processes a dir.
     958             :  */
     959         219 : static void scan_emptydir(struct snapraid_scan* scan, const char* sub)
     960             : {
     961         219 :         struct snapraid_disk* disk = scan->disk;
     962             :         struct snapraid_dir* dir;
     963             : 
     964             :         /* check if the dir already exists */
     965         219 :         dir = tommy_hashdyn_search(&disk->dirset, dir_name_compare, sub, dir_name_hash(sub));
     966         219 :         if (dir) {
     967             :                 /* check if multiple files have the same name */
     968         209 :                 if (dir_flag_has(dir, FILE_IS_PRESENT)) {
     969             :                         /* LCOV_EXCL_START */
     970             :                         log_fatal("Internal inconsistency for dir '%s%s'\n", disk->dir, sub);
     971             :                         exit(EXIT_FAILURE);
     972             :                         /* LCOV_EXCL_STOP */
     973             :                 }
     974             : 
     975             :                 /* mark as present */
     976         209 :                 dir_flag_set(dir, FILE_IS_PRESENT);
     977             : 
     978             :                 /* nothing more to do */
     979         428 :                 return;
     980             :         } else {
     981             :                 /* and continue to insert it */
     982             :         }
     983             : 
     984             :         /* insert it */
     985          10 :         dir = dir_alloc(sub);
     986             : 
     987             :         /* mark it as present */
     988          10 :         dir_flag_set(dir, FILE_IS_PRESENT);
     989             : 
     990             :         /* insert it in the delayed insert list */
     991          10 :         tommy_list_insert_tail(&scan->dir_insert_list, &dir->nodelist, dir);
     992             : }
     993             : 
     994             : struct dirent_sorted {
     995             :         /* node for data structures */
     996             :         tommy_node node;
     997             : 
     998             : #if HAVE_STRUCT_DIRENT_D_INO
     999             :         uint64_t d_ino; /**< Inode number. */
    1000             : #endif
    1001             : #if HAVE_STRUCT_DIRENT_D_TYPE
    1002             :         uint32_t d_type; /**< File type. */
    1003             : #endif
    1004             : #if HAVE_STRUCT_DIRENT_D_STAT
    1005             :         struct stat d_stat; /**< Stat result. */
    1006             : #endif
    1007             :         char d_name[]; /**< Variable length name. It must be the last field. */
    1008             : };
    1009             : 
    1010             : #if HAVE_STRUCT_DIRENT_D_INO
    1011      103300 : static int dd_ino_compare(const void* void_a, const void* void_b)
    1012             : {
    1013      103300 :         const struct dirent_sorted* a = void_a;
    1014      103300 :         const struct dirent_sorted* b = void_b;
    1015             : 
    1016      103300 :         if (a->d_ino < b->d_ino)
    1017       49394 :                 return -1;
    1018       53906 :         if (a->d_ino > b->d_ino)
    1019       53906 :                 return 1;
    1020             : 
    1021           0 :         return 0;
    1022             : }
    1023             : #endif
    1024             : 
    1025    10206597 : static int dd_name_compare(const void* void_a, const void* void_b)
    1026             : {
    1027    10206597 :         const struct dirent_sorted* a = void_a;
    1028    10206597 :         const struct dirent_sorted* b = void_b;
    1029             : 
    1030    10206597 :         return strcmp(a->d_name, b->d_name);
    1031             : }
    1032             : 
    1033             : /**
    1034             :  * Returns the stat info of a dir entry.
    1035             :  */
    1036             : #if HAVE_STRUCT_DIRENT_D_STAT
    1037             : #define DSTAT(file, dd, buf) dstat(dd)
    1038             : struct stat* dstat(struct dirent_sorted* dd)
    1039             : {
    1040             :         return &dd->d_stat;
    1041             : }
    1042             : #else
    1043             : #define DSTAT(file, dd, buf) dstat(file, buf)
    1044      976055 : struct stat* dstat(const char* file, struct stat* st)
    1045             : {
    1046      976055 :         if (lstat(file, st) != 0) {
    1047             :                 /* LCOV_EXCL_START */
    1048             :                 log_fatal("Error in stat file/directory '%s'. %s.\n", file, strerror(errno));
    1049             :                 exit(EXIT_FAILURE);
    1050             :                 /* LCOV_EXCL_STOP */
    1051             :         }
    1052      976055 :         return st;
    1053             : }
    1054             : #endif
    1055             : 
    1056             : /**
    1057             :  * Processes a directory.
    1058             :  * Return != 0 if at least one file or link is processed.
    1059             :  */
    1060        1502 : static int scan_dir(struct snapraid_scan* scan, int is_diff, const char* dir, const char* sub)
    1061             : {
    1062        1502 :         struct snapraid_state* state = scan->state;
    1063        1502 :         struct snapraid_disk* disk = scan->disk;
    1064        1502 :         int processed = 0;
    1065             :         DIR* d;
    1066             :         tommy_list list;
    1067             :         tommy_node* node;
    1068             : 
    1069        1502 :         tommy_list_init(&list);
    1070             : 
    1071        1502 :         d = opendir(dir);
    1072        1502 :         if (!d) {
    1073             :                 /* LCOV_EXCL_START */
    1074             :                 log_fatal("Error opening directory '%s'. %s.\n", dir, strerror(errno));
    1075             :                 log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub);
    1076             :                 exit(EXIT_FAILURE);
    1077             :                 /* LCOV_EXCL_STOP */
    1078             :         }
    1079             : 
    1080             :         /* read the full directory */
    1081             :         while (1) {
    1082             :                 char path_next[PATH_MAX];
    1083             :                 char sub_next[PATH_MAX];
    1084             :                 struct dirent_sorted* entry;
    1085             :                 const char* name;
    1086             :                 struct dirent* dd;
    1087             :                 size_t name_len;
    1088             : 
    1089             :                 /* clear errno to detect erroneous conditions */
    1090     1012101 :                 errno = 0;
    1091     1012101 :                 dd = readdir(d);
    1092     1012101 :                 if (dd == 0 && errno != 0) {
    1093             :                         /* LCOV_EXCL_START */
    1094             :                         log_fatal("Error reading directory '%s'. %s.\n", dir, strerror(errno));
    1095             :                         log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub);
    1096             :                         exit(EXIT_FAILURE);
    1097             :                         /* LCOV_EXCL_STOP */
    1098             :                 }
    1099     1012101 :                 if (dd == 0) {
    1100        1502 :                         break; /* finished */
    1101             :                 }
    1102             : 
    1103             :                 /* skip "." and ".." files */
    1104     1010599 :                 name = dd->d_name;
    1105     1010599 :                 if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
    1106        6008 :                         continue;
    1107             : 
    1108     1007595 :                 pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
    1109     1007595 :                 pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
    1110             : 
    1111             :                 /* check for not supported file names */
    1112     1007595 :                 if (name[0] == 0) {
    1113             :                         /* LCOV_EXCL_START */
    1114             :                         log_fatal("Unsupported name '%s' in file '%s'.\n", name, path_next);
    1115             :                         exit(EXIT_FAILURE);
    1116             :                         /* LCOV_EXCL_STOP */
    1117             :                 }
    1118             : 
    1119             :                 /* exclude hidden files even before calling lstat() */
    1120     1007595 :                 if (filter_hidden(state->filter_hidden, dd) != 0) {
    1121           0 :                         msg_verbose("Excluding hidden '%s'\n", path_next);
    1122           0 :                         continue;
    1123             :                 }
    1124             : 
    1125             :                 /* exclude content files even before calling lstat() */
    1126     1007595 :                 if (filter_content(&state->contentlist, path_next) != 0) {
    1127           0 :                         msg_verbose("Excluding content '%s'\n", path_next);
    1128           0 :                         continue;
    1129             :                 }
    1130             : 
    1131     1007595 :                 name_len = strlen(dd->d_name);
    1132     1007595 :                 entry = malloc_nofail(sizeof(struct dirent_sorted) + name_len + 1);
    1133             : 
    1134             :                 /* copy the dir entry */
    1135             : #if HAVE_STRUCT_DIRENT_D_INO
    1136     1007595 :                 entry->d_ino = dd->d_ino;
    1137             : #endif
    1138             : #if HAVE_STRUCT_DIRENT_D_TYPE
    1139     1007595 :                 entry->d_type = dd->d_type;
    1140             : #endif
    1141             : #if HAVE_STRUCT_DIRENT_D_STAT
    1142             :                 /* convert dirent to lstat result */
    1143             :                 dirent_lstat(dd, &entry->d_stat);
    1144             : 
    1145             :                 /* note that at this point the st_mode may be 0 */
    1146             : #endif
    1147     1007595 :                 memcpy(entry->d_name, dd->d_name, name_len + 1);
    1148             : 
    1149             :                 /* insert in the list */
    1150     1007595 :                 tommy_list_insert_tail(&list, &entry->node, entry);
    1151     1010599 :         }
    1152             : 
    1153        1502 :         if (closedir(d) != 0) {
    1154             :                 /* LCOV_EXCL_START */
    1155             :                 log_fatal("Error closing directory '%s'. %s.\n", dir, strerror(errno));
    1156             :                 exit(EXIT_FAILURE);
    1157             :                 /* LCOV_EXCL_STOP */
    1158             :         }
    1159             : 
    1160        1502 :         if (state->opt.force_order == SORT_ALPHA) {
    1161             :                 /* if requested sort alphabetically */
    1162             :                 /* this is mainly done for testing to ensure to always */
    1163             :                 /* process in the same way in different platforms */
    1164        1466 :                 tommy_list_sort(&list, dd_name_compare);
    1165             :         }
    1166             : #if HAVE_STRUCT_DIRENT_D_INO
    1167          36 :         else if (!disk->has_volatile_inodes) {
    1168             :                 /* if inodes are persistent */
    1169             :                 /* sort the list of dir entries by inodes */
    1170          36 :                 tommy_list_sort(&list, dd_ino_compare);
    1171             :         }
    1172             :         /* otherwise just keep the insertion order */
    1173             : #endif
    1174             : 
    1175             :         /* process the sorted dir entries */
    1176        1502 :         node = list;
    1177     1010599 :         while (node != 0) {
    1178             :                 char path_next[PATH_MAX];
    1179             :                 char sub_next[PATH_MAX];
    1180             :                 char out[PATH_MAX];
    1181     1007595 :                 struct snapraid_filter* reason = 0;
    1182     1007595 :                 struct dirent_sorted* dd = node->data;
    1183     1007595 :                 const char* name = dd->d_name;
    1184             :                 struct stat* st;
    1185             :                 int type;
    1186             : #if !HAVE_STRUCT_DIRENT_D_STAT
    1187             :                 struct stat st_buf;
    1188             : #endif
    1189             : 
    1190     1007595 :                 pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
    1191     1007595 :                 pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
    1192             : 
    1193             :                 /* start with an unknown type */
    1194     1007595 :                 type = -1;
    1195     1007595 :                 st = 0;
    1196             : 
    1197             :                 /* if dirent has the type, use it */
    1198             : #if HAVE_STRUCT_DIRENT_D_TYPE
    1199     1007595 :                 switch (dd->d_type) {
    1200           0 :                 case DT_UNKNOWN : break;
    1201      974966 :                 case DT_REG : type = 0; break;
    1202       31540 :                 case DT_LNK : type = 1; break;
    1203        1089 :                 case DT_DIR : type = 2; break;
    1204           0 :                 default : type = 3; break;
    1205             :                 }
    1206             : #endif
    1207             : 
    1208             :                 /* if type is still unknown */
    1209     1007595 :                 if (type < 0) {
    1210             :                         /* get the type from stat */
    1211           0 :                         st = DSTAT(path_next, dd, &st_buf);
    1212             : 
    1213             : #if HAVE_STRUCT_DIRENT_D_STAT
    1214             :                         /* if the st_mode field is missing, takes care to fill it using normal lstat() */
    1215             :                         /* at now this can happen only in Windows (with HAVE_STRUCT_DIRENT_D_STAT defined), */
    1216             :                         /* because we use a directory reading method that doesn't read info about ReparsePoint. */
    1217             :                         /* Note that here we cannot call here lstat_sync(), because we don't know what kind */
    1218             :                         /* of file is it, and lstat_sync() doesn't always work */
    1219             :                         if (st->st_mode == 0) {
    1220             :                                 if (lstat(path_next, st) != 0) {
    1221             :                                         /* LCOV_EXCL_START */
    1222             :                                         log_fatal("Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
    1223             :                                         exit(EXIT_FAILURE);
    1224             :                                         /* LCOV_EXCL_STOP */
    1225             :                                 }
    1226             :                         }
    1227             : #endif
    1228             : 
    1229           0 :                         if (S_ISREG(st->st_mode))
    1230           0 :                                 type = 0;
    1231           0 :                         else if (S_ISLNK(st->st_mode))
    1232           0 :                                 type = 1;
    1233           0 :                         else if (S_ISDIR(st->st_mode))
    1234           0 :                                 type = 2;
    1235             :                         else
    1236           0 :                                 type = 3;
    1237             :                 }
    1238             : 
    1239     1007595 :                 if (type == 0) { /* REG */
    1240      974966 :                         if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
    1241             : 
    1242             :                                 /* late stat, if not yet called */
    1243      974966 :                                 if (!st)
    1244      974966 :                                         st = DSTAT(path_next, dd, &st_buf);
    1245             : 
    1246             : #if HAVE_LSTAT_SYNC
    1247             :                                 /* if the st_ino field is missing, takes care to fill it using the extended lstat() */
    1248             :                                 /* this can happen only in Windows */
    1249             :                                 if (st->st_ino == 0) {
    1250             :                                         if (lstat_sync(path_next, st, 0) != 0) {
    1251             :                                                 /* LCOV_EXCL_START */
    1252             :                                                 log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
    1253             :                                                 exit(EXIT_FAILURE);
    1254             :                                                 /* LCOV_EXCL_STOP */
    1255             :                                         }
    1256             :                                 }
    1257             : #endif
    1258             : 
    1259      974966 :                                 scan_file(scan, is_diff, sub_next, st, FILEPHY_UNREAD_OFFSET);
    1260      974966 :                                 processed = 1;
    1261             :                         } else {
    1262           0 :                                 msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
    1263             :                         }
    1264       32629 :                 } else if (type == 1) { /* LNK */
    1265       31540 :                         if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
    1266             :                                 char subnew[PATH_MAX];
    1267             :                                 int ret;
    1268             : 
    1269       31540 :                                 ret = readlink(path_next, subnew, sizeof(subnew));
    1270       31540 :                                 if (ret >= PATH_MAX) {
    1271             :                                         /* LCOV_EXCL_START */
    1272             :                                         log_fatal("Error in readlink file '%s'. Symlink too long.\n", path_next);
    1273             :                                         exit(EXIT_FAILURE);
    1274             :                                         /* LCOV_EXCL_STOP */
    1275             :                                 }
    1276       31540 :                                 if (ret < 0) {
    1277             :                                         /* LCOV_EXCL_START */
    1278             :                                         log_fatal("Error in readlink file '%s'. %s.\n", path_next, strerror(errno));
    1279             :                                         exit(EXIT_FAILURE);
    1280             :                                         /* LCOV_EXCL_STOP */
    1281             :                                 }
    1282             : 
    1283             :                                 /* readlink doesn't put the final 0 */
    1284       31540 :                                 subnew[ret] = 0;
    1285             : 
    1286             :                                 /* process as a symbolic link */
    1287       31540 :                                 scan_link(scan, is_diff, sub_next, subnew, FILE_IS_SYMLINK);
    1288       31540 :                                 processed = 1;
    1289             :                         } else {
    1290           0 :                                 msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
    1291             :                         }
    1292        1089 :                 } else if (type == 2) { /* DIR */
    1293        1089 :                         if (filter_dir(&state->filterlist, &reason, disk->name, sub_next) == 0) {
    1294             : #ifndef _WIN32
    1295             :                                 /* late stat, if not yet called */
    1296        1089 :                                 if (!st)
    1297        1089 :                                         st = DSTAT(path_next, dd, &st_buf);
    1298             : 
    1299             :                                 /* in Unix don't follow mount points in different devices */
    1300             :                                 /* in Windows we are already skipping them reporting them as special files */
    1301        1089 :                                 if ((uint64_t)st->st_dev != disk->device) {
    1302           0 :                                         log_fatal("WARNING! Ignoring mount point '%s' because it appears to be in a different device\n", path_next);
    1303             :                                 } else
    1304             : #endif
    1305             :                                 {
    1306             :                                         char sub_dir[PATH_MAX];
    1307             : 
    1308             :                                         /* recurse */
    1309        1089 :                                         pathslash(path_next, sizeof(path_next));
    1310        1089 :                                         pathcpy(sub_dir, sizeof(sub_dir), sub_next);
    1311        1089 :                                         pathslash(sub_dir, sizeof(sub_dir));
    1312        1089 :                                         if (scan_dir(scan, is_diff, path_next, sub_dir) == 0) {
    1313             :                                                 /* scan the directory as empty dir */
    1314         219 :                                                 scan_emptydir(scan, sub_next);
    1315             :                                         }
    1316             :                                         /* or we processed something internally, or we have added the empty dir */
    1317        1089 :                                         processed = 1;
    1318             :                                 }
    1319             :                         } else {
    1320           0 :                                 msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
    1321             :                         }
    1322             :                 } else {
    1323           0 :                         if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
    1324             :                                 /* late stat, if not yet called */
    1325           0 :                                 if (!st)
    1326           0 :                                         st = DSTAT(path_next, dd, &st_buf);
    1327             : 
    1328           0 :                                 log_fatal("WARNING! Ignoring special '%s' file '%s'\n", stat_desc(st), path_next);
    1329             :                         } else {
    1330           0 :                                 msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
    1331             :                         }
    1332             :                 }
    1333             : 
    1334             :                 /* next entry */
    1335     1007595 :                 node = node->next;
    1336             : 
    1337             :                 /* free the present one */
    1338     1007595 :                 free(dd);
    1339             :         }
    1340             : 
    1341        1502 :         return processed;
    1342             : }
    1343             : 
    1344          69 : static int state_diffscan(struct snapraid_state* state, int is_diff)
    1345             : {
    1346             :         tommy_node* i;
    1347             :         tommy_node* j;
    1348             :         tommy_list scanlist;
    1349             :         int done;
    1350             :         fptr* msg;
    1351             :         struct snapraid_scan total;
    1352             :         int no_difference;
    1353             : 
    1354          69 :         tommy_list_init(&scanlist);
    1355             : 
    1356          69 :         if (is_diff)
    1357           6 :                 msg_progress("Comparing...\n");
    1358             : 
    1359             :         /* first scan all the directory and find new and deleted files */
    1360         482 :         for (i = state->disklist; i != 0; i = i->next) {
    1361         413 :                 struct snapraid_disk* disk = i->data;
    1362             :                 struct snapraid_scan* scan;
    1363             :                 tommy_node* node;
    1364             :                 int ret;
    1365             :                 int has_persistent_inode;
    1366             : 
    1367         413 :                 scan = malloc_nofail(sizeof(struct snapraid_scan));
    1368         413 :                 scan->state = state;
    1369         413 :                 scan->disk = disk;
    1370         413 :                 scan->count_equal = 0;
    1371         413 :                 scan->count_move = 0;
    1372         413 :                 scan->count_copy = 0;
    1373         413 :                 scan->count_restore = 0;
    1374         413 :                 scan->count_change = 0;
    1375         413 :                 scan->count_remove = 0;
    1376         413 :                 scan->count_insert = 0;
    1377         413 :                 tommy_list_init(&scan->file_insert_list);
    1378         413 :                 tommy_list_init(&scan->link_insert_list);
    1379         413 :                 tommy_list_init(&scan->dir_insert_list);
    1380             : 
    1381         413 :                 tommy_list_insert_tail(&scanlist, &scan->node, scan);
    1382             : 
    1383         413 :                 if (!is_diff)
    1384         377 :                         msg_progress("Scanning disk %s...\n", disk->name);
    1385             : 
    1386             :                 /* check if the disk supports persistent inodes */
    1387         413 :                 ret = fsinfo(disk->dir, &has_persistent_inode, 0, 0);
    1388         413 :                 if (ret < 0) {
    1389             :                         /* LCOV_EXCL_START */
    1390             :                         log_fatal("Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno));
    1391             :                         exit(EXIT_FAILURE);
    1392             :                         /* LCOV_EXCL_STOP */
    1393             :                 }
    1394         413 :                 if (!has_persistent_inode) {
    1395           0 :                         disk->has_volatile_inodes = 1;
    1396             :                 }
    1397             : 
    1398             :                 /* if inodes or UUID are not persistent/changed/unsupported */
    1399         413 :                 if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
    1400             :                         /* removes all the inodes from the inode collection */
    1401             :                         /* if they are not persistent, all of them could be changed now */
    1402             :                         /* and we don't want to find false matching ones */
    1403             :                         /* see scan_file() for more details */
    1404          37 :                         node = disk->filelist;
    1405          74 :                         while (node) {
    1406           0 :                                 struct snapraid_file* file = node->data;
    1407             : 
    1408           0 :                                 node = node->next;
    1409             : 
    1410             :                                 /* remove from the inode set */
    1411           0 :                                 tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
    1412             : 
    1413             :                                 /* clear the inode */
    1414           0 :                                 file->inode = 0;
    1415             : 
    1416             :                                 /* mark as missing inode */
    1417           0 :                                 file_flag_set(file, FILE_IS_WITHOUT_INODE);
    1418             :                         }
    1419             :                 }
    1420             : 
    1421         413 :                 scan_dir(scan, is_diff, disk->dir, "");
    1422             :         }
    1423             : 
    1424             :         /* we split the search in two phases because to detect files */
    1425             :         /* moved from one disk to another we have to start deletion */
    1426             :         /* only when all disks have all the new files found */
    1427             : 
    1428             :         /* now process all the new and deleted files */
    1429         482 :         for (i = scanlist; i != 0; i = i->next) {
    1430         413 :                 struct snapraid_scan* scan = i->data;
    1431         413 :                 struct snapraid_disk* disk = scan->disk;
    1432             :                 tommy_node* node;
    1433             :                 unsigned phy_count;
    1434             :                 unsigned phy_dup;
    1435             :                 uint64_t phy_last;
    1436             :                 struct snapraid_file* phy_file_last;
    1437             : 
    1438             :                 /* check for removed files */
    1439         413 :                 node = disk->filelist;
    1440      905053 :                 while (node) {
    1441      904227 :                         struct snapraid_file* file = node->data;
    1442             : 
    1443             :                         /* next node */
    1444      904227 :                         node = node->next;
    1445             : 
    1446             :                         /* remove if not present */
    1447      904227 :                         if (!file_flag_has(file, FILE_IS_PRESENT)) {
    1448       29629 :                                 ++scan->count_remove;
    1449             : 
    1450       29629 :                                 log_tag("scan:remove:%s:%s\n", disk->name, esc(file->sub));
    1451       29629 :                                 if (is_diff) {
    1452          97 :                                         printf("remove %s%s\n", disk->dir, file->sub);
    1453             :                                 }
    1454             : 
    1455       29629 :                                 scan_file_remove(scan, file);
    1456             :                         }
    1457             :                 }
    1458             : 
    1459             :                 /* check for removed links */
    1460         413 :                 node = disk->linklist;
    1461       31191 :                 while (node) {
    1462       30365 :                         struct snapraid_link* link = node->data;
    1463             : 
    1464             :                         /* next node */
    1465       30365 :                         node = node->next;
    1466             : 
    1467             :                         /* remove if not present */
    1468       30365 :                         if (!link_flag_has(link, FILE_IS_PRESENT)) {
    1469         956 :                                 ++scan->count_remove;
    1470             : 
    1471         956 :                                 log_tag("scan:remove:%s:%s\n", disk->name, esc(link->sub));
    1472         956 :                                 if (is_diff) {
    1473           9 :                                         printf("remove %s%s\n", disk->dir, link->sub);
    1474             :                                 }
    1475             : 
    1476         956 :                                 scan_link_remove(scan, link);
    1477             :                         }
    1478             :                 }
    1479             : 
    1480             :                 /* check for removed dirs */
    1481         413 :                 node = disk->dirlist;
    1482        1037 :                 while (node) {
    1483         211 :                         struct snapraid_dir* dir = node->data;
    1484             : 
    1485             :                         /* next node */
    1486         211 :                         node = node->next;
    1487             : 
    1488             :                         /* remove if not present */
    1489         211 :                         if (!dir_flag_has(dir, FILE_IS_PRESENT)) {
    1490           2 :                                 scan_emptydir_remove(scan, dir);
    1491             :                         }
    1492             :                 }
    1493             : 
    1494             :                 /* sort the files before inserting them */
    1495             :                 /* we use a stable sort to ensure that if the reported physical offset/inode */
    1496             :                 /* are always 0, we keep at least the directory order */
    1497         413 :                 switch (state->opt.force_order) {
    1498             :                 case SORT_PHYSICAL :
    1499          12 :                         tommy_list_sort(&scan->file_insert_list, file_physical_compare);
    1500          12 :                         break;
    1501             :                 case SORT_INODE :
    1502           0 :                         tommy_list_sort(&scan->file_insert_list, file_inode_compare);
    1503           0 :                         break;
    1504             :                 case SORT_ALPHA :
    1505         401 :                         tommy_list_sort(&scan->file_insert_list, file_path_compare);
    1506         401 :                         break;
    1507             :                 case SORT_DIR :
    1508             :                         /* already in order */
    1509           0 :                         break;
    1510             :                 }
    1511             : 
    1512             :                 /* insert all the new files, we insert them only after the deletion */
    1513             :                 /* to reuse the just freed space */
    1514             :                 /* also check if the physical offset reported are fakes or not */
    1515         413 :                 node = scan->file_insert_list;
    1516         413 :                 phy_count = 0;
    1517         413 :                 phy_dup = 0;
    1518         413 :                 phy_last = FILEPHY_UNREAD_OFFSET;
    1519         413 :                 phy_file_last = 0;
    1520      100868 :                 while (node) {
    1521      100042 :                         struct snapraid_file* file = node->data;
    1522             : 
    1523             :                         /* if the file is not empty, count duplicate physical offsets */
    1524      100042 :                         if (state->opt.force_order == SORT_PHYSICAL && file->size != 0) {
    1525       11418 :                                 if (phy_file_last != 0 && file->physical == phy_last
    1526             :                                         /* files without offset are expected to have duplicates */
    1527           0 :                                         && phy_last != FILEPHY_WITHOUT_OFFSET
    1528             :                                 ) {
    1529             :                                         /* if verbose, prints the list of duplicates real offsets */
    1530             :                                         /* other cases are for offsets not supported, so we don't need to report them file by file */
    1531           0 :                                         if (phy_last >= FILEPHY_REAL_OFFSET) {
    1532           0 :                                                 log_fatal("WARNING! Files '%s%s' and '%s%s' have the same physical offset %" PRId64 ".\n", disk->dir, phy_file_last->sub, disk->dir, file->sub, phy_last);
    1533             :                                         }
    1534           0 :                                         ++phy_dup;
    1535             :                                 }
    1536       11418 :                                 phy_file_last = file;
    1537       11418 :                                 phy_last = file->physical;
    1538       11418 :                                 ++phy_count;
    1539             :                         }
    1540             : 
    1541             :                         /* next node */
    1542      100042 :                         node = node->next;
    1543             : 
    1544             :                         /* insert it */
    1545      100042 :                         scan_file_insert(scan, file);
    1546             :                 }
    1547             : 
    1548             :                 /* mark the disk without reliable physical offset if it has duplicates */
    1549             :                 /* here it should never happen because we already sorted out hardlinks */
    1550         413 :                 if (state->opt.force_order == SORT_PHYSICAL && phy_dup > 0) {
    1551           0 :                         disk->has_unreliable_physical = 1;
    1552             :                 }
    1553             : 
    1554             :                 /* insert all the new links */
    1555         413 :                 node = scan->link_insert_list;
    1556        3283 :                 while (node) {
    1557        2457 :                         struct snapraid_link* link = node->data;
    1558             : 
    1559             :                         /* next node */
    1560        2457 :                         node = node->next;
    1561             : 
    1562             :                         /* insert it */
    1563        2457 :                         scan_link_insert(scan, link);
    1564             :                 }
    1565             : 
    1566             :                 /* insert all the new dirs */
    1567         413 :                 node = scan->dir_insert_list;
    1568         836 :                 while (node) {
    1569          10 :                         struct snapraid_dir* dir = node->data;
    1570             : 
    1571             :                         /* next node */
    1572          10 :                         node = node->next;
    1573             : 
    1574             :                         /* insert it */
    1575          10 :                         scan_emptydir_insert(scan, dir);
    1576             :                 }
    1577             :         }
    1578             : 
    1579             :         /* checks for disks where all the previously existing files where removed */
    1580          69 :         if (!state->opt.force_empty) {
    1581          68 :                 int all_missing = 0;
    1582          68 :                 int all_rewritten = 0;
    1583          68 :                 done = 0;
    1584         475 :                 for (i = state->disklist, j = scanlist; i != 0; i = i->next, j = j->next) {
    1585         407 :                         struct snapraid_disk* disk = i->data;
    1586         407 :                         struct snapraid_scan* scan = j->data;
    1587             : 
    1588         407 :                         if (scan->count_equal == 0
    1589          37 :                                 && scan->count_move == 0
    1590          37 :                                 && scan->count_restore == 0
    1591          37 :                                 && (scan->count_remove != 0 || scan->count_change != 0)
    1592             :                         ) {
    1593           0 :                                 if (!done) {
    1594           0 :                                         done = 1;
    1595           0 :                                         log_fatal("WARNING! All the files previously present in disk '%s' at dir '%s'", disk->name, disk->dir);
    1596             :                                 } else {
    1597           0 :                                         log_fatal(", disk '%s' at dir '%s'", disk->name, disk->dir);
    1598             :                                 }
    1599             : 
    1600             :                                 /* detect the special condition of all files missing */
    1601           0 :                                 if (scan->count_change == 0)
    1602           0 :                                         all_missing = 1;
    1603             : 
    1604             :                                 /* detect the special condition of all files rewritten */
    1605           0 :                                 if (scan->count_remove == 0)
    1606           0 :                                         all_rewritten = 1;
    1607             :                         }
    1608             :                 }
    1609          68 :                 if (done) {
    1610           0 :                         log_fatal("\nare now missing or rewritten!\n");
    1611           0 :                         if (all_rewritten) {
    1612           0 :                                 log_fatal("This could happen when restoring a disk with a backup\n");
    1613           0 :                                 log_fatal("program that is not setting correctly the timestamps.\n");
    1614             :                         }
    1615           0 :                         if (all_missing) {
    1616           0 :                                 log_fatal("This could happen when some disks are not mounted\n");
    1617           0 :                                 log_fatal("in the expected directory.\n");
    1618             :                         }
    1619           0 :                         if (!is_diff) {
    1620           0 :                                 log_fatal("If you want to '%s' anyway, use 'snapraid --force-empty %s'.", state->command, state->command);
    1621           0 :                                 exit(EXIT_FAILURE);
    1622             :                         }
    1623             :                 }
    1624             :         }
    1625             : 
    1626             :         /* checks for disks without the physical offset support */
    1627          69 :         if (state->opt.force_order == SORT_PHYSICAL) {
    1628           2 :                 done = 0;
    1629          14 :                 for (i = state->disklist; i != 0; i = i->next) {
    1630          12 :                         struct snapraid_disk* disk = i->data;
    1631             : 
    1632          12 :                         if (disk->has_unreliable_physical) {
    1633           0 :                                 if (!done) {
    1634           0 :                                         done = 1;
    1635           0 :                                         log_fatal("WARNING! Physical offsets not supported for disk '%s'", disk->name);
    1636             :                                 } else {
    1637           0 :                                         log_fatal(", '%s'", disk->name);
    1638             :                                 }
    1639             :                         }
    1640             :                 }
    1641           2 :                 if (done) {
    1642           0 :                         log_fatal(". Files order won't be optimal.\n");
    1643             :                 }
    1644             :         }
    1645             : 
    1646             :         /* checks for disks without persistent inodes */
    1647          69 :         done = 0;
    1648         482 :         for (i = state->disklist; i != 0; i = i->next) {
    1649         413 :                 struct snapraid_disk* disk = i->data;
    1650             : 
    1651         413 :                 if (disk->has_volatile_inodes) {
    1652           0 :                         if (!done) {
    1653           0 :                                 done = 1;
    1654           0 :                                 log_fatal("WARNING! Inodes are not persistent for disks: '%s'", disk->name);
    1655             :                         } else {
    1656           0 :                                 log_fatal(", '%s'", disk->name);
    1657             :                         }
    1658             :                 }
    1659             :         }
    1660          69 :         if (done) {
    1661           0 :                 log_fatal(". Move operations won't be optimal.\n");
    1662             :         }
    1663             : 
    1664             :         /* checks for disks with changed UUID */
    1665          69 :         done = 0;
    1666         482 :         for (i = state->disklist; i != 0; i = i->next) {
    1667         413 :                 struct snapraid_disk* disk = i->data;
    1668             : 
    1669             :                 /* don't print the message if the UUID changed because before */
    1670             :                 /* it was no set. */
    1671             :                 /* this is the normal condition for an empty disk because it */
    1672             :                 /* isn't stored */
    1673         413 :                 if (disk->has_different_uuid && !disk->had_empty_uuid) {
    1674           0 :                         if (!done) {
    1675           0 :                                 done = 1;
    1676           0 :                                 log_fatal("WARNING! UUID is changed for disks: '%s'", disk->name);
    1677             :                         } else {
    1678           0 :                                 log_fatal(", '%s'", disk->name);
    1679             :                         }
    1680             :                 }
    1681             :         }
    1682          69 :         if (done) {
    1683           0 :                 log_fatal(". Move operations won't be optimal.\n");
    1684             :         }
    1685             : 
    1686             :         /* checks for disks with unsupported UUID */
    1687          69 :         done = 0;
    1688         482 :         for (i = state->disklist; i != 0; i = i->next) {
    1689         413 :                 struct snapraid_disk* disk = i->data;
    1690             : 
    1691         413 :                 if (disk->has_unsupported_uuid) {
    1692           0 :                         if (!done) {
    1693           0 :                                 done = 1;
    1694           0 :                                 log_fatal("WARNING! UUID is unsupported for disks: '%s'", disk->name);
    1695             :                         } else {
    1696           0 :                                 log_fatal(", '%s'", disk->name);
    1697             :                         }
    1698             :                 }
    1699             :         }
    1700          69 :         if (done) {
    1701           0 :                 log_fatal(". Move operations won't be optimal.\n");
    1702             :         }
    1703             : 
    1704          69 :         total.count_equal = 0;
    1705          69 :         total.count_move = 0;
    1706          69 :         total.count_copy = 0;
    1707          69 :         total.count_restore = 0;
    1708          69 :         total.count_change = 0;
    1709          69 :         total.count_remove = 0;
    1710          69 :         total.count_insert = 0;
    1711             : 
    1712         482 :         for (i = scanlist; i != 0; i = i->next) {
    1713         413 :                 struct snapraid_scan* scan = i->data;
    1714         413 :                 total.count_equal += scan->count_equal;
    1715         413 :                 total.count_move += scan->count_move;
    1716         413 :                 total.count_copy += scan->count_copy;
    1717         413 :                 total.count_restore += scan->count_restore;
    1718         413 :                 total.count_change += scan->count_change;
    1719         413 :                 total.count_remove += scan->count_remove;
    1720         413 :                 total.count_insert += scan->count_insert;
    1721             :         }
    1722             : 
    1723          69 :         if (is_diff) {
    1724           6 :                 msg_status("\n");
    1725           6 :                 msg = msg_status;
    1726             :         } else {
    1727          63 :                 msg = msg_verbose;
    1728             :         }
    1729             : 
    1730          69 :         msg("%8u equal\n", total.count_equal);
    1731          69 :         msg("%8u moved\n", total.count_move);
    1732          69 :         msg("%8u copied\n", total.count_copy);
    1733          69 :         msg("%8u restored\n", total.count_restore);
    1734          69 :         msg("%8u updated\n", total.count_change);
    1735          69 :         msg("%8u removed\n", total.count_remove);
    1736          69 :         msg("%8u added\n", total.count_insert);
    1737             : 
    1738          69 :         log_tag("summary:equal:%u\n", total.count_equal);
    1739          69 :         log_tag("summary:moved:%u\n", total.count_move);
    1740          69 :         log_tag("summary:copied:%u\n", total.count_copy);
    1741          69 :         log_tag("summary:restored:%u\n", total.count_restore);
    1742          69 :         log_tag("summary:updated:%u\n", total.count_change);
    1743          69 :         log_tag("summary:removed:%u\n", total.count_remove);
    1744          69 :         log_tag("summary:added:%u\n", total.count_insert);
    1745             : 
    1746         206 :         no_difference = !total.count_move && !total.count_copy && !total.count_restore
    1747         109 :                 && !total.count_change && !total.count_remove && !total.count_insert;
    1748             : 
    1749          69 :         if (is_diff) {
    1750           6 :                 if (no_difference) {
    1751           1 :                         msg_status("No differences\n");
    1752             :                 } else {
    1753           5 :                         msg_status("There are differences!\n");
    1754             :                 }
    1755             :         }
    1756             : 
    1757          69 :         if (no_difference) {
    1758           7 :                 log_tag("summary:exit:equal\n");
    1759             :         } else {
    1760          62 :                 log_tag("summary:exit:diff\n");
    1761             :         }
    1762          69 :         log_flush();
    1763             : 
    1764          69 :         tommy_list_foreach(&scanlist, (tommy_foreach_func*)free);
    1765             : 
    1766          69 :         if (is_diff) {
    1767             :                 /* check for file difference */
    1768           6 :                 if (!no_difference)
    1769           5 :                         return 1;
    1770             : 
    1771             :                 /* check also for incomplete "sync" */
    1772           1 :                 if (parity_is_invalid(state))
    1773           0 :                         return 1;
    1774             :         }
    1775             : 
    1776          64 :         return 0;
    1777             : }
    1778             : 
    1779           6 : int state_diff(struct snapraid_state* state)
    1780             : {
    1781           6 :         return state_diffscan(state, 1);
    1782             : }
    1783             : 
    1784          63 : void state_scan(struct snapraid_state* state)
    1785             : {
    1786          63 :         (void)state_diffscan(state, 0); /* ignore return value */
    1787          63 : }
    1788             : 

Generated by: LCOV version 1.10