LCOV - code coverage report
Current view: top level - cmdline - scan.c (source / functions) Hit Total Coverage
Test: lcov.info Lines: 565 631 89.5 %
Date: 2016-11-21 Functions: 23 23 100.0 %

          Line data    Source code
       1             : /*
       2             :  * Copyright (C) 2011 Andrea Mazzoleni
       3             :  *
       4             :  * This program is free software: you can redistribute it and/or modify
       5             :  * it under the terms of the GNU General Public License as published by
       6             :  * the Free Software Foundation, either version 3 of the License, or
       7             :  * (at your option) any later version.
       8             :  *
       9             :  * This program is distributed in the hope that it will be useful,
      10             :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      12             :  * GNU General Public License for more details.
      13             :  *
      14             :  * You should have received a copy of the GNU General Public License
      15             :  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "portable.h"
      19             : 
      20             : #include "support.h"
      21             : #include "elem.h"
      22             : #include "state.h"
      23             : #include "parity.h"
      24             : 
      25             : struct snapraid_scan {
      26             :         struct snapraid_state* state; /**< State used. */
      27             :         struct snapraid_disk* disk; /**< Disk used. */
      28             : 
      29             :         /**
      30             :          * Counters of changes.
      31             :          */
      32             :         unsigned count_equal; /**< Files equal. */
      33             :         unsigned count_move; /**< Files with a different name, but equal inode, size and timestamp in the same disk. */
      34             :         unsigned count_restore; /**< Files with equal name, size and timestamp, but different inode. */
      35             :         unsigned count_change; /**< Files with same name, but different size and/or timestamp. */
      36             :         unsigned count_copy; /**< Files new, with same name size and timestamp of a file in a different disk. */
      37             :         unsigned count_insert; /**< Files new. */
      38             :         unsigned count_remove; /**< Files removed. */
      39             : 
      40             :         tommy_list file_insert_list; /**< Files to insert. */
      41             :         tommy_list link_insert_list; /**< Links to insert. */
      42             :         tommy_list dir_insert_list; /**< Dirs to insert. */
      43             : 
      44             :         /* nodes for data structures */
      45             :         tommy_node node;
      46             : };
      47             : 
      48             : /**
      49             :  * Remove the specified link from the data set.
      50             :  */
      51         956 : static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* slink)
      52             : {
      53         956 :         struct snapraid_state* state = scan->state;
      54         956 :         struct snapraid_disk* disk = scan->disk;
      55             : 
      56             :         /* state changed */
      57         956 :         state->need_write = 1;
      58             : 
      59             :         /* remove the file from the link containers */
      60         956 :         tommy_hashdyn_remove_existing(&disk->linkset, &slink->nodeset);
      61         956 :         tommy_list_remove_existing(&disk->linklist, &slink->nodelist);
      62             : 
      63             :         /* deallocate */
      64         956 :         link_free(slink);
      65         956 : }
      66             : 
      67             : /**
      68             :  * Insert the specified link in the data set.
      69             :  */
      70        2457 : static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* slink)
      71             : {
      72        2457 :         struct snapraid_state* state = scan->state;
      73        2457 :         struct snapraid_disk* disk = scan->disk;
      74             : 
      75             :         /* state changed */
      76        2457 :         state->need_write = 1;
      77             : 
      78             :         /* insert the link in the link containers */
      79        2457 :         tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
      80        2457 :         tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
      81        2457 : }
      82             : 
      83             : /**
      84             :  * Process a symbolic link.
      85             :  */
      86       36184 : static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub, const char* linkto, unsigned link_flag)
      87             : {
      88       36184 :         struct snapraid_state* state = scan->state;
      89       36184 :         struct snapraid_disk* disk = scan->disk;
      90             :         struct snapraid_link* slink;
      91             :         char esc_buffer[ESC_MAX];
      92             : 
      93             :         /* check if the link already exists */
      94       36184 :         slink = tommy_hashdyn_search(&disk->linkset, link_name_compare_to_arg, sub, link_name_hash(sub));
      95       36184 :         if (slink) {
      96             :                 /* check if multiple files have the same name */
      97       33727 :                 if (link_flag_has(slink, FILE_IS_PRESENT)) {
      98             :                         /* LCOV_EXCL_START */
      99             :                         log_fatal("Internal inconsistency for link '%s%s'\n", disk->dir, sub);
     100             :                         os_abort();
     101             :                         /* LCOV_EXCL_STOP */
     102             :                 }
     103             : 
     104             :                 /* mark as present */
     105       33727 :                 link_flag_set(slink, FILE_IS_PRESENT);
     106             : 
     107             :                 /* check if the link is not changed and it's of the same kind */
     108       33727 :                 if (strcmp(slink->linkto, linkto) == 0 && link_flag == link_flag_get(slink, FILE_IS_LINK_MASK)) {
     109             :                         /* it's equal */
     110       33721 :                         ++scan->count_equal;
     111             : 
     112       67442 :                         if (state->opt.gui) {
     113         829 :                                 log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
     114             :                         }
     115             :                 } else {
     116             :                         /* it's an update */
     117             : 
     118             :                         /* we have to save the linkto/type */
     119           6 :                         state->need_write = 1;
     120             : 
     121           6 :                         ++scan->count_change;
     122             : 
     123           6 :                         log_tag("scan:update:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
     124           6 :                         if (is_diff) {
     125           3 :                                 printf("update %s\n", fmt_term(disk, slink->sub, esc_buffer));
     126             :                         }
     127             : 
     128             :                         /* update it */
     129           6 :                         free(slink->linkto);
     130           6 :                         slink->linkto = strdup_nofail(linkto);
     131           6 :                         link_flag_let(slink, link_flag, FILE_IS_LINK_MASK);
     132             :                 }
     133             : 
     134             :                 /* nothing more to do */
     135       69911 :                 return;
     136             :         } else {
     137             :                 /* create the new link */
     138        2457 :                 ++scan->count_insert;
     139             : 
     140        2457 :                 log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
     141        2457 :                 if (is_diff) {
     142         599 :                         printf("add %s\n", fmt_term(disk, sub, esc_buffer));
     143             :                 }
     144             : 
     145             :                 /* and continue to insert it */
     146             :         }
     147             : 
     148             :         /* insert it */
     149        2457 :         slink = link_alloc(sub, linkto, link_flag);
     150             : 
     151             :         /* mark it as present */
     152        2457 :         link_flag_set(slink, FILE_IS_PRESENT);
     153             : 
     154             :         /* insert it in the delayed insert list */
     155        2457 :         tommy_list_insert_tail(&scan->link_insert_list, &slink->nodelist, slink);
     156             : }
     157             : 
     158             : /**
     159             :  * Insert the specified file in the parity.
     160             :  */
     161      101881 : static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
     162             : {
     163      101881 :         struct snapraid_state* state = scan->state;
     164      101881 :         struct snapraid_disk* disk = scan->disk;
     165             :         block_off_t i;
     166             :         block_off_t parity_pos;
     167             : 
     168             :         /* state changed */
     169      101881 :         state->need_write = 1;
     170             : 
     171             :         /* allocate the blocks of the file */
     172      101881 :         parity_pos = disk->first_free_block;
     173      353073 :         for (i = 0; i < file->blockmax; ++i) {
     174             :                 struct snapraid_block* block;
     175             :                 struct snapraid_block* over_block;
     176             :                 snapraid_info info;
     177             : 
     178             :                 /* increment the position until the first really free block */
     179     1022715 :                 while (block_has_file(fs_par2block_find(disk, parity_pos)))
     180      520331 :                         ++parity_pos;
     181             : 
     182             :                 /* get block we are going to overwrite, if any */
     183      251192 :                 over_block = fs_par2block_find(disk, parity_pos);
     184             : 
     185             :                 /* deallocate it */
     186      251192 :                 if (over_block != BLOCK_NULL)
     187       95279 :                         fs_deallocate(disk, parity_pos);
     188             : 
     189             :                 /* get block specific info */
     190      251192 :                 info = info_get(&state->infoarr, parity_pos);
     191             : 
     192             :                 /* get the new block we are going to write */
     193      251192 :                 block = fs_file2block_get(file, i);
     194             : 
     195             :                 /* if the file block already has an updated hash without rehash */
     196      251192 :                 if (block_has_updated_hash(block) && !info_get_rehash(info)) {
     197             :                         /* the only possible case is for REP blocks */
     198       86215 :                         assert(block_state_get(block) == BLOCK_STATE_REP);
     199             : 
     200             :                         /* convert to a REP block */
     201       86215 :                         block_state_set(block, BLOCK_STATE_REP);
     202             : 
     203             :                         /* and keep the hash as it's */
     204             :                 } else {
     205             :                         unsigned over_state;
     206             : 
     207             :                         /* convert to a CHG block */
     208      164977 :                         block_state_set(block, BLOCK_STATE_CHG);
     209             : 
     210             :                         /* state of the block we are going to overwrite */
     211      164977 :                         over_state = block_state_get(over_block);
     212             : 
     213             :                         /* if the block is an empty one */
     214      164977 :                         if (over_state == BLOCK_STATE_EMPTY) {
     215             :                                 /* the block was empty and filled with zeros */
     216             :                                 /* set the hash to the special ZERO value */
     217      115543 :                                 hash_zero_set(block->hash);
     218             :                         } else {
     219             :                                 /* otherwise it's a DELETED one */
     220       49434 :                                 assert(over_state == BLOCK_STATE_DELETED);
     221             : 
     222             :                                 /* copy the past hash of the block */
     223       49434 :                                 memcpy(block->hash, over_block->hash, BLOCK_HASH_SIZE);
     224             : 
     225             :                                 /* if we have not already cleared the past hash */
     226       49434 :                                 if (!state->clear_past_hash) {
     227             :                                         /* in this case we don't know if the old state is still the one */
     228             :                                         /* stored inside the parity, because after an aborted sync, the parity */
     229             :                                         /* may be or may be not have been updated with the new data */
     230             :                                         /* Then we reset the hash to a bogus value */
     231             :                                         /* For example: */
     232             :                                         /* - One file is deleted */
     233             :                                         /* - Sync aborted after, updating the parity to the new state, */
     234             :                                         /*   but without saving the content file representing this new state. */
     235             :                                         /* - Another file is added again (exactly here) */
     236             :                                         /*   with the hash of DELETED block not representing the real parity state */
     237         421 :                                         hash_invalid_set(block->hash);
     238             :                                 }
     239             :                         }
     240             :                 }
     241             : 
     242             :                 /* store in the disk map, after invalidating all the other blocks */
     243      251192 :                 fs_allocate(disk, parity_pos, file, i);
     244             : 
     245             :                 /* set the new free position */
     246      251192 :                 disk->first_free_block = parity_pos + 1;
     247             :         }
     248             : 
     249             :         /* insert in the list of contained files */
     250      101881 :         tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
     251      101881 : }
     252             : 
     253             : /**
     254             :  * Delete the specified file from the parity.
     255             :  *
     256             :  * Note that the parity remains allocated, but the blocks and the file are marked as DELETED.
     257             :  * The file is then inserted in the deleted set, and it should not be deallocated,
     258             :  * as the parity still references it.
     259             :  */
     260       57015 : static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
     261             : {
     262       57015 :         struct snapraid_state* state = scan->state;
     263       57015 :         struct snapraid_disk* disk = scan->disk;
     264             :         block_off_t i;
     265             : 
     266             :         /* remove from the list of contained files */
     267       57015 :         tommy_list_remove_existing(&disk->filelist, &file->nodelist);
     268             : 
     269             :         /* state changed */
     270       57015 :         state->need_write = 1;
     271             : 
     272             :         /* here we are supposed to adjust the ::first_free_block position */
     273             :         /* with the parity position we are deleting */
     274             :         /* but we also know that we do only delayed insert, after all the deletion, */
     275             :         /* so at this point ::first_free_block is always at 0, and we don't need to update it */
     276       57015 :         if (disk->first_free_block != 0) {
     277             :                 /* LCOV_EXCL_START */
     278             :                 log_fatal("Internal inconsistency for first free position at '%u' deallocating file '%s'\n", disk->first_free_block, file->sub);
     279             :                 os_abort();
     280             :                 /* LCOV_EXCL_STOP */
     281             :         }
     282             : 
     283             :         /* free all the blocks of the file */
     284      197592 :         for (i = 0; i < file->blockmax; ++i) {
     285      140577 :                 struct snapraid_block* block = fs_file2block_get(file, i);
     286             :                 unsigned block_state;
     287             : 
     288             :                 /* in case we scan after an aborted sync, */
     289             :                 /* we could get also intermediate states */
     290      140577 :                 block_state = block_state_get(block);
     291      140577 :                 switch (block_state) {
     292             :                 case BLOCK_STATE_BLK :
     293             :                         /* we keep the hash making it an "old" hash, because the parity is still containing data for it */
     294       35580 :                         break;
     295             :                 case BLOCK_STATE_CHG :
     296             :                         /* if we have not already cleared the past hash */
     297       28040 :                         if (!state->clear_past_hash) {
     298             :                                 /* in these cases we don't know if the old state is still the one */
     299             :                                 /* stored inside the parity, because after an aborted sync, the parity */
     300             :                                 /* may be or may be not have been updated with the data that it's now */
     301             :                                 /* deleted. Then we reset the hash to a bogus value. */
     302             :                                 /* For example: */
     303             :                                 /* - One file is added */
     304             :                                 /* - Sync aborted after updating the parity to the new state, */
     305             :                                 /*   but without saving the content file representing this new state. */
     306             :                                 /* - File is now deleted after the aborted sync */
     307             :                                 /* - Sync again, deleting the blocks (exactly here) */
     308             :                                 /*   with the hash of CHG block not representing the real parity state */
     309          40 :                                 hash_invalid_set(block->hash);
     310             :                         }
     311       28040 :                         break;
     312             :                 case BLOCK_STATE_REP :
     313             :                         /* we just don't know the old hash, and then we set it to invalid */
     314       76957 :                         hash_invalid_set(block->hash);
     315       76957 :                         break;
     316             :                 default :
     317             :                         /* LCOV_EXCL_START */
     318             :                         log_fatal("Internal inconsistency in file '%s' deallocating block '%u:%u' state %u\n", file->sub, i, file->blockmax, block_state);
     319             :                         os_abort();
     320             :                         /* LCOV_EXCL_STOP */
     321             :                 }
     322             : 
     323             :                 /* set the block as deleted */
     324      140577 :                 block_state_set(block, BLOCK_STATE_DELETED);
     325             :         }
     326             : 
     327             :         /* mark the file as deleted */
     328       57015 :         file_flag_set(file, FILE_IS_DELETED);
     329             : 
     330             :         /* insert it in the list of deleted blocks */
     331       57015 :         tommy_list_insert_tail(&disk->deletedlist, &file->nodelist, file);
     332       57015 : }
     333             : 
     334      101881 : static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
     335             : {
     336      101881 :         struct snapraid_state* state = scan->state;
     337      101881 :         struct snapraid_disk* disk = scan->disk;
     338             : 
     339             :         /* if we sort for physical offsets we have to read them for new files */
     340      101881 :         if (state->opt.force_order == SORT_PHYSICAL
     341       11432 :                 && file->physical == FILEPHY_UNREAD_OFFSET
     342             :         ) {
     343             :                 char path_next[PATH_MAX];
     344             : 
     345       11432 :                 pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, file->sub);
     346             : 
     347       11432 :                 if (filephy(path_next, file->size, &file->physical) != 0) {
     348             :                         /* LCOV_EXCL_START */
     349             :                         log_fatal("Error in getting the physical offset of file '%s'. %s.\n", path_next, strerror(errno));
     350             :                         exit(EXIT_FAILURE);
     351             :                         /* LCOV_EXCL_STOP */
     352             :                 }
     353             :         }
     354             : 
     355             :         /* insert in the delayed list */
     356      101881 :         tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
     357      101881 : }
     358             : 
     359             : /**
     360             :  * Check if a file is completely formed of blocks with invalid parity,
     361             :  * and no rehash is tagged, and if it has at least one block.
     362             :  */
     363     1025550 : static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
     364             : {
     365             :         block_off_t i;
     366             : 
     367             :         /* with no block, it never has an invalid parity */
     368     1025550 :         if (file->blockmax == 0)
     369        1439 :                 return 0;
     370             : 
     371             :         /* check all blocks */
     372     1080079 :         for (i = 0; i < file->blockmax; ++i) {
     373             :                 snapraid_info info;
     374     1057392 :                 struct snapraid_block* block = fs_file2block_get(file, i);
     375             :                 block_off_t parity_pos;
     376             : 
     377             :                 /* exclude blocks with parity */
     378     1057392 :                 if (!block_has_invalid_parity(block))
     379     1001424 :                         return 0;
     380             : 
     381             :                 /*
     382             :                  * Get the parity position.
     383             :                  *
     384             :                  * Note that here we expect to always have mapped
     385             :                  * parity, because kept files always have it.
     386             :                  *
     387             :                  * Anyway, checking for POS_NULL doesn't hurt.
     388             :                  */
     389       55968 :                 parity_pos = fs_file2par_find(disk, file, i);
     390             : 
     391             :                 /* if it's not mapped, it cannot have rehash */
     392       55968 :                 if (parity_pos != POS_NULL) {
     393             :                         /* get block specific info */
     394       55968 :                         info = info_get(&state->infoarr, parity_pos);
     395             : 
     396             :                         /* if rehash fails */
     397       55968 :                         if (info_get_rehash(info))
     398           0 :                                 return 0;
     399             :                 }
     400             :         }
     401             : 
     402       22687 :         return 1;
     403             : }
     404             : 
     405             : /**
     406             :  * Check if a file is completely formed of blocks with an updated hash,
     407             :  * and no rehash is tagged, and if it has at least one block.
     408             :  */
     409       23670 : static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
     410             : {
     411             :         block_off_t i;
     412             : 
     413             :         /* with no block, it never has a hash */
     414       23670 :         if (file->blockmax == 0)
     415          27 :                 return 0;
     416             : 
     417             :         /* check all blocks */
     418       81886 :         for (i = 0; i < file->blockmax; ++i) {
     419             :                 snapraid_info info;
     420       58250 :                 struct snapraid_block* block = fs_file2block_get(file, i);
     421             :                 block_off_t parity_pos;
     422             : 
     423             :                 /* exclude blocks without hash */
     424       58250 :                 if (!block_has_updated_hash(block))
     425           7 :                         return 0;
     426             : 
     427             :                 /*
     428             :                  * Get the parity position.
     429             :                  *
     430             :                  * Note that it's possible to have files
     431             :                  * not mapped into the parity, even if they
     432             :                  * have a valid hash.
     433             :                  *
     434             :                  * This happens for example, for 'copied' files
     435             :                  * that have REP blocks, but not yet mapped.
     436             :                  *
     437             :                  * If there are multiple copies, it's also possible
     438             :                  * that such files are used as 'source' to copy
     439             :                  * hashes, and then to get them inside this function.
     440             :                  */
     441       58243 :                 parity_pos = fs_file2par_find(disk, file, i);
     442             : 
     443             :                 /* if it's not mapped, it cannot have rehash */
     444       58243 :                 if (parity_pos != POS_NULL) {
     445             :                         /* get block specific info */
     446       58235 :                         info = info_get(&state->infoarr, parity_pos);
     447             : 
     448             :                         /* exclude blocks needing a rehash */
     449       58235 :                         if (info_get_rehash(info))
     450           0 :                                 return 0;
     451             :                 }
     452             :         }
     453             : 
     454       23636 :         return 1;
     455             : }
     456             : 
     457             : /**
     458             :  * Refresh the file info.
     459             :  *
     460             :  * This is needed by Windows as the normal way to list directories may report not
     461             :  * updated info. Only the GetFileInformationByHandle() func, called file-by-file,
     462             :  * really ensures to return synced info.
     463             :  *
     464             :  * If this happens, we read also the physical offset, to avoid to read it later.
     465             :  */
     466       79194 : static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struct stat* st, uint64_t* physical)
     467             : {
     468             : #if HAVE_LSTAT_SYNC
     469             :         struct snapraid_state* state = scan->state;
     470             :         struct snapraid_disk* disk = scan->disk;
     471             : 
     472             :         /* if the st_sync is not set, ensure to get synced info */
     473             :         if (st->st_sync == 0) {
     474             :                 char path_next[PATH_MAX];
     475             :                 struct stat synced_st;
     476             : 
     477             :                 pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, sub);
     478             : 
     479             :                 /* if we sort for physical offsets we have to read them for new files */
     480             :                 if (state->opt.force_order == SORT_PHYSICAL
     481             :                         && *physical == FILEPHY_UNREAD_OFFSET
     482             :                 ) {
     483             :                         /* do nothing, leave the pointer to read the physical offset */
     484             :                 } else {
     485             :                         physical = 0; /* set the pointer to 0 to read nothing */
     486             :                 }
     487             : 
     488             :                 if (lstat_sync(path_next, &synced_st, physical) != 0) {
     489             :                         /* LCOV_EXCL_START */
     490             :                         log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
     491             :                         exit(EXIT_FAILURE);
     492             :                         /* LCOV_EXCL_STOP */
     493             :                 }
     494             : 
     495             :                 if (st->st_mtime != synced_st.st_mtime
     496             :                         || st->st_mtimensec != synced_st.st_mtimensec
     497             :                 ) {
     498             : #ifndef _WIN32
     499             :                         /*
     500             :                          * In Windows having different metadata is expected with open files
     501             :                          * because the metadata in the directory is updated only when the file
     502             :                          * is closed.
     503             :                          *
     504             :                          * See also:
     505             :                          * Why is the file size reported incorrectly for files that are still being written to?
     506             :                          * http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx
     507             :                          */
     508             :                         log_fatal("WARNING! Detected uncached time change for file '%s'\n", sub);
     509             :                         log_fatal("It's better if you run SnapRAID without other processes running.\n");
     510             : #endif
     511             :                         st->st_mtime = synced_st.st_mtime;
     512             :                         st->st_mtimensec = synced_st.st_mtimensec;
     513             :                 }
     514             : 
     515             :                 if (st->st_size != synced_st.st_size) {
     516             : #ifndef _WIN32
     517             :                         log_fatal("WARNING! Detected uncached size change for file '%s'\n", sub);
     518             :                         log_fatal("It's better if you run SnapRAID without other processes running.\n");
     519             : #endif
     520             :                         st->st_size = synced_st.st_size;
     521             :                 }
     522             : 
     523             :                 if (st->st_ino != synced_st.st_ino) {
     524             :                         log_fatal("DANGER! Detected uncached inode change for file '%s'\n", sub);
     525             :                         log_fatal("Please ensure to run SnapRAID without other processes running.\n");
     526             :                         /* at this point, it's too late to change inode */
     527             :                         /* and having inconsistent inodes may result to internal failures */
     528             :                         /* so, it's better to abort */
     529             :                         exit(EXIT_FAILURE);
     530             :                 }
     531             :         }
     532             : #else
     533             :         (void)scan;
     534             :         (void)sub;
     535             :         (void)st;
     536             :         (void)physical;
     537             : #endif
     538       79194 : }
     539             : 
     540             : /**
     541             :  * Insert the file in the data set.
     542             :  */
     543      101881 : static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
     544             : {
     545      101881 :         struct snapraid_disk* disk = scan->disk;
     546             : 
     547             :         /* insert the file in the containers */
     548      101881 :         if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
     549      101881 :                 tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
     550      101881 :         tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
     551      101881 :         tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
     552             : 
     553             :         /* delayed allocation of the parity */
     554      101881 :         scan_file_delayed_allocate(scan, file);
     555      101881 : }
     556             : 
     557             : /**
     558             :  * Remove the file from the data set.
     559             :  *
     560             :  * File is then deleted.
     561             :  */
     562       57015 : static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
     563             : {
     564       57015 :         struct snapraid_disk* disk = scan->disk;
     565             : 
     566             :         /* remove the file from the containers */
     567       57015 :         if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
     568       56941 :                 tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
     569       57015 :         tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
     570       57015 :         tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
     571             : 
     572             :         /* deallocate the file from the parity */
     573       57015 :         scan_file_deallocate(scan, file);
     574       57015 : }
     575             : 
     576             : /**
     577             :  * Keep the file as it's (or with only a name/inode modification).
     578             :  *
     579             :  * If the file is kept, nothing has to be done.
     580             :  *
     581             :  * But if a file contains only blocks with invalid parity, it's reallocated to ensure
     582             :  * to always minimize the space used in the parity.
     583             :  *
     584             :  * This could happen after a failed sync, when some other files are deleted,
     585             :  * and then new ones can be moved backward to fill the hole created.
     586             :  */
     587     1025550 : static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* file)
     588             : {
     589     1025550 :         struct snapraid_disk* disk = scan->disk;
     590             : 
     591             :         /* if the file is full invalid, schedule a reinsert at later stage */
     592     1025550 :         if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) {
     593       22687 :                 struct snapraid_file* copy = file_dup(file);
     594             : 
     595             :                 /* remove the file */
     596       22687 :                 scan_file_remove(scan, file);
     597             : 
     598             :                 /* reinsert the copy in the delayed list */
     599       22687 :                 scan_file_insert(scan, copy);
     600             :         }
     601     1025550 : }
     602             : 
     603             : /**
     604             :  * Process a file.
     605             :  */
     606     1104939 : static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, struct stat* st, uint64_t physical)
     607             : {
     608     1104939 :         struct snapraid_state* state = scan->state;
     609     1104939 :         struct snapraid_disk* disk = scan->disk;
     610             :         struct snapraid_file* file;
     611             :         tommy_node* i;
     612             :         int is_original_file_size_different_than_zero;
     613             :         int is_file_already_present;
     614             :         data_off_t file_already_present_size;
     615             :         int64_t file_already_present_mtime_sec;
     616             :         int file_already_present_mtime_nsec;
     617             :         int is_file_reported;
     618             :         char esc_buffer[ESC_MAX];
     619             :         char esc_buffer_alt[ESC_MAX];
     620             : 
     621             :         /*
     622             :          * If the disk has persistent inodes and UUID, try a search on the past inodes,
     623             :          * to detect moved files.
     624             :          *
     625             :          * For persistent inodes we mean inodes that keep their values when the file-system
     626             :          * is unmounted and remounted. This don't always happen.
     627             :          *
     628             :          * Cases found are:
     629             :          * - Linux FUSE with exFAT driver from https://code.google.com/p/exfat/.
     630             :          *   Inodes are reassigned at every mount restarting from 1 and incrementing.
     631             :          *   As worse, the exFAT support in FUSE doesn't use sub-second precision in timestamps
     632             :          *   making inode collision more easy (exFAT by design supports 10ms precision).
     633             :          * - Linux VFAT kernel (3.2) driver. Inodes are fully reassigned at every mount.
     634             :          *
     635             :          * In such cases, to avoid possible random collisions, it's better to disable the moved
     636             :          * file recognition.
     637             :          *
     638             :          * For persistent UUID we mean that it has the same UUID as before.
     639             :          * Otherwise, if the UUID is changed, likely it's a new recreated file-system,
     640             :          * and then the inode have no meaning.
     641             :          *
     642             :          * Note that to disable the search by past inode, we do this implicitly
     643             :          * removing all the past inode before searching for files.
     644             :          * This ensures that no file is found with a past inode, but at the same time,
     645             :          * it allows to find new files with the same inode, to identify them as hardlinks.
     646             :          */
     647     1104939 :         int has_past_inodes = !disk->has_volatile_inodes && !disk->has_different_uuid && !disk->has_unsupported_uuid;
     648             : 
     649             :         /* always search with the new inode, in the all new inodes found until now, */
     650             :         /* with the eventual presence of also the past inodes */
     651     1104939 :         uint64_t inode = st->st_ino;
     652             : 
     653     1104939 :         file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
     654             : 
     655             :         /* identify moved files with past inodes and hardlinks with the new inodes */
     656     1104939 :         if (file) {
     657             :                 /* check if the file is not changed */
     658      943901 :                 if (file->size == st->st_size
     659      935613 :                         && file->mtime_sec == st->st_mtime
     660      932556 :                         && (file->mtime_nsec == STAT_NSEC(st)
     661             :                         /* always accept the stored value if it's STAT_NSEC_INVALID */
     662             :                         /* it happens when upgrading from an old version of SnapRAID */
     663             :                         /* not yet supporting the nanosecond field */
     664           1 :                         || file->mtime_nsec == STAT_NSEC_INVALID
     665             :                         )
     666             :                 ) {
     667             :                         /* check if multiple files have the same inode */
     668      932555 :                         if (file_flag_has(file, FILE_IS_PRESENT)) {
     669             : #if HAVE_STRUCT_STAT_ST_NLINK
     670         195 :                                 if (st->st_nlink <= 1) {
     671             :                                         /* LCOV_EXCL_START */
     672             :                                         log_fatal("Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub);
     673             :                                         os_abort();
     674             :                                         /* LCOV_EXCL_STOP */
     675             :                                 }
     676             : #endif
     677             :                                 /* it's a hardlink */
     678         195 :                                 scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
     679         195 :                                 return;
     680             :                         }
     681             : 
     682             :                         /* mark as present */
     683      932360 :                         file_flag_set(file, FILE_IS_PRESENT);
     684             : 
     685             :                         /* update the nanoseconds mtime only if different */
     686             :                         /* to avoid unneeded updates */
     687      932360 :                         if (file->mtime_nsec == STAT_NSEC_INVALID
     688           0 :                                 && STAT_NSEC(st) != file->mtime_nsec
     689             :                         ) {
     690           0 :                                 file->mtime_nsec = STAT_NSEC(st);
     691             : 
     692             :                                 /* we have to save the new mtime */
     693           0 :                                 state->need_write = 1;
     694             :                         }
     695             : 
     696      932360 :                         if (strcmp(file->sub, sub) != 0) {
     697             :                                 /* if the path is different, it means a moved file with the same inode */
     698         155 :                                 ++scan->count_move;
     699             : 
     700         155 :                                 log_tag("scan:move:%s:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer), esc_tag(sub, esc_buffer_alt));
     701         155 :                                 if (is_diff) {
     702           0 :                                         printf("move %s -> %s\n", fmt_term(disk, file->sub, esc_buffer), fmt_term(disk, sub, esc_buffer_alt));
     703             :                                 }
     704             : 
     705             :                                 /* remove from the name set */
     706         155 :                                 tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
     707             : 
     708             :                                 /* save the new name */
     709         155 :                                 file_rename(file, sub);
     710             : 
     711             :                                 /* reinsert in the name set */
     712         155 :                                 tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
     713             : 
     714             :                                 /* we have to save the new name */
     715         155 :                                 state->need_write = 1;
     716             :                         } else {
     717             :                                 /* otherwise it's equal */
     718      932205 :                                 ++scan->count_equal;
     719             : 
     720      932205 :                                 if (state->opt.gui) {
     721       22611 :                                         log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
     722             :                                 }
     723             :                         }
     724             : 
     725             :                         /* mark the file as kept */
     726      932360 :                         scan_file_keep(scan, file);
     727             : 
     728             :                         /* nothing more to do */
     729      932360 :                         return;
     730             :                 }
     731             : 
     732             :                 /* here the file matches the inode, but not the other info */
     733             :                 /* if could be a modified file with the same name, */
     734             :                 /* or a restored/copied file that get assigned a previously used inode, */
     735             :                 /* or a file-system with not persistent inodes */
     736             : 
     737             :                 /* for sure it cannot be already present */
     738       11346 :                 if (file_flag_has(file, FILE_IS_PRESENT)) {
     739             :                         /* LCOV_EXCL_START */
     740             :                         log_fatal("Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' matching and already present but different\n", file->inode, disk->dir, sub, disk->dir, file->sub);
     741             :                         os_abort();
     742             :                         /* LCOV_EXCL_STOP */
     743             :                 }
     744             : 
     745             :                 /* assume a previously used inode, it's the worst case */
     746             :                 /* and we handle it removing the duplicate stored inode. */
     747             :                 /* If the file is found by name later, it will have the inode restored, */
     748             :                 /* otherwise, it will get removed */
     749             : 
     750             :                 /* remove from the inode set */
     751       11346 :                 tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
     752             : 
     753             :                 /* clear the inode */
     754             :                 /* this is not really needed for correct functionality */
     755             :                 /* because we are going to set FILE_IS_WITHOUT_INODE */
     756             :                 /* but it's easier for debugging to have invalid inodes set to 0 */
     757       11346 :                 file->inode = 0;
     758             : 
     759             :                 /* mark as missing inode */
     760       11346 :                 file_flag_set(file, FILE_IS_WITHOUT_INODE);
     761             : 
     762             :                 /* go further to find it by name */
     763             :         }
     764             : 
     765             :         /* initialize for later overwrite */
     766      172384 :         is_file_reported = 0;
     767      172384 :         is_original_file_size_different_than_zero = 0;
     768             : 
     769             :         /* then try finding it by name */
     770      172384 :         file = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub, file_path_hash(sub));
     771             : 
     772             :         /* keep track if the file already exists */
     773      172384 :         is_file_already_present = file != 0;
     774             : 
     775      172384 :         if (is_file_already_present) {
     776             :                 /* if the file is without an inode */
     777       97871 :                 if (file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
     778             :                         /* set it now */
     779       11286 :                         file->inode = st->st_ino;
     780             : 
     781             :                         /* insert in the set */
     782       11286 :                         tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
     783             : 
     784             :                         /* unmark as missing inode */
     785       11286 :                         file_flag_clear(file, FILE_IS_WITHOUT_INODE);
     786             :                 } else {
     787             :                         /* here the inode has to be different, otherwise we would have found it before */
     788       86585 :                         if (file->inode == st->st_ino) {
     789             :                                 /* LCOV_EXCL_START */
     790             :                                 log_fatal("Internal inconsistency in inode '%" PRIu64 "' for files '%s%s' as unexpected matching\n", file->inode, disk->dir, sub);
     791             :                                 os_abort();
     792             :                                 /* LCOV_EXCL_STOP */
     793             :                         }
     794             :                 }
     795             : 
     796             :                 /* for sure it cannot be already present */
     797       97871 :                 if (file_flag_has(file, FILE_IS_PRESENT)) {
     798             :                         /* LCOV_EXCL_START */
     799             :                         log_fatal("Internal inconsistency in path for file '%s%s' matching and already present\n", disk->dir, sub);
     800             :                         os_abort();
     801             :                         /* LCOV_EXCL_STOP */
     802             :                 }
     803             : 
     804             :                 /* check if the file is not changed */
     805       97871 :                 if (file->size == st->st_size
     806       97275 :                         && file->mtime_sec == st->st_mtime
     807       93190 :                         && (file->mtime_nsec == STAT_NSEC(st)
     808             :                         /* always accept the stored value if it's STAT_NSEC_INVALID */
     809             :                         /* it happens when upgrading from an old version of SnapRAID */
     810             :                         /* not yet supporting the nanosecond field */
     811           0 :                         || file->mtime_nsec == STAT_NSEC_INVALID
     812             :                         )
     813             :                 ) {
     814             :                         /* mark as present */
     815       93190 :                         file_flag_set(file, FILE_IS_PRESENT);
     816             : 
     817             :                         /* update the nano seconds mtime only if different */
     818             :                         /* to avoid unneeded updates */
     819       93190 :                         if (file->mtime_nsec == STAT_NSEC_INVALID
     820           0 :                                 && STAT_NSEC(st) != STAT_NSEC_INVALID
     821             :                         ) {
     822           0 :                                 file->mtime_nsec = STAT_NSEC(st);
     823             : 
     824             :                                 /* we have to save the new mtime */
     825           0 :                                 state->need_write = 1;
     826             :                         }
     827             : 
     828             :                         /* if when processing the disk we used the past inodes values */
     829       93190 :                         if (has_past_inodes) {
     830             :                                 /* if persistent inodes are supported, we are sure that the inode number */
     831             :                                 /* is now different, because otherwise the file would have been found */
     832             :                                 /* when searching by inode. */
     833             :                                 /* if the inode is different, it means a rewritten file with the same path */
     834             :                                 /* like when restoring a backup that restores also the timestamp */
     835       93176 :                                 ++scan->count_restore;
     836             : 
     837       93176 :                                 log_tag("scan:restore:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
     838       93176 :                                 if (is_diff) {
     839           0 :                                         printf("restore %s\n", fmt_term(disk, sub, esc_buffer));
     840             :                                 }
     841             : 
     842             :                                 /* remove from the inode set */
     843       93176 :                                 tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
     844             : 
     845             :                                 /* save the new inode */
     846       93176 :                                 file->inode = st->st_ino;
     847             : 
     848             :                                 /* reinsert in the inode set */
     849       93176 :                                 tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
     850             : 
     851             :                                 /* we have to save the new inode */
     852       93176 :                                 state->need_write = 1;
     853             :                         } else {
     854             :                                 /* otherwise it's the case of not persistent inode, where doesn't */
     855             :                                 /* matter if the inode is different or equal, because they have no */
     856             :                                 /* meaning, and then we don't even save them */
     857          14 :                                 ++scan->count_equal;
     858             : 
     859          14 :                                 if (state->opt.gui) {
     860           0 :                                         log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
     861             :                                 }
     862             :                         }
     863             : 
     864             :                         /* mark the file as kept */
     865       93190 :                         scan_file_keep(scan, file);
     866             : 
     867             :                         /* nothing more to do */
     868       93190 :                         return;
     869             :                 }
     870             : 
     871             :                 /* here if the file is changed but with the correct name */
     872             : 
     873             :                 /* save the info for later printout */
     874        4681 :                 file_already_present_size = file->size;
     875        4681 :                 file_already_present_mtime_sec = file->mtime_sec;
     876        4681 :                 file_already_present_mtime_nsec = file->mtime_nsec;
     877             : 
     878             :                 /* keep track if the original file was not of zero size */
     879        4681 :                 is_original_file_size_different_than_zero = file->size != 0;
     880             : 
     881             :                 /* remove it, and continue to insert it again */
     882        4681 :                 scan_file_remove(scan, file);
     883             : 
     884             :                 /* and continue to insert it again */
     885             :         } else {
     886       74513 :                 file_already_present_size = 0;
     887       74513 :                 file_already_present_mtime_sec = 0;
     888       74513 :                 file_already_present_mtime_nsec = 0;
     889             :         }
     890             : 
     891             :         /* refresh the info, to ensure that they are synced, */
     892             :         /* note that we refresh only the info of the new or modified files */
     893             :         /* because this is slow operation */
     894       79194 :         scan_file_refresh(scan, sub, st, &physical);
     895             : 
     896             : #ifndef _WIN32
     897             :         /* do a safety check to ensure that the common ext4 case of zeroing */
     898             :         /* the size of a file after a crash doesn't propagate to the backup */
     899             :         /* this check is specific for Linux, so we disable it on Windows */
     900       79194 :         if (is_original_file_size_different_than_zero && st->st_size == 0) {
     901           0 :                 if (!state->opt.force_zero) {
     902             :                         /* LCOV_EXCL_START */
     903             :                         log_fatal("The file '%s%s' has unexpected zero size!\n", disk->dir, sub);
     904             :                         log_fatal("It's possible that after a kernel crash this file was lost,\n");
     905             :                         log_fatal("and you can use 'snapraid fix -f /%s' to recover it.\n", fmt_poll(disk, sub, esc_buffer));
     906             :                         if (!is_diff) {
     907             :                                 log_fatal("If this an expected condition you can '%s' anyway using 'snapraid --force-zero %s'\n", state->command, state->command);
     908             :                                 exit(EXIT_FAILURE);
     909             :                         }
     910             :                         /* LCOV_EXCL_STOP */
     911             :                 }
     912             :         }
     913             : #else
     914             :         /* avoid the unused warning in Windows */
     915             :         (void)is_original_file_size_different_than_zero;
     916             : #endif
     917             : 
     918             :         /* insert it */
     919       79194 :         file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, STAT_NSEC(st), st->st_ino, physical);
     920             : 
     921             :         /* mark it as present */
     922       79194 :         file_flag_set(file, FILE_IS_PRESENT);
     923             : 
     924             :         /* if copy detection is enabled */
     925             :         /* note that the copy detection is tried also for updated files */
     926             :         /* this makes sense because it may happen to have two different copies */
     927             :         /* of the same file, and we move the right one over the wrong one */
     928             :         /* in such case we have a "copy" over an "update" */
     929       79194 :         if (!state->opt.force_nocopy) {
     930       79194 :                 tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
     931             : 
     932             :                 /* search for a file with the same name and stamp in all the disks */
     933      423410 :                 for (i = state->disklist; i != 0; i = i->next) {
     934      367852 :                         struct snapraid_disk* other_disk = i->data;
     935             :                         struct snapraid_file* other_file;
     936             : 
     937             :                         /* if the nanosecond part of the time stamp is valid, search */
     938             :                         /* for name and stamp, otherwise for path and stamp */
     939      367852 :                         if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
     940      367744 :                                 other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
     941             :                         else
     942         108 :                                 other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
     943             : 
     944             :                         /* if found, and it's a fully hashed file */
     945      367852 :                         if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
     946             :                                 /* assume that the file is a copy, and reuse the hash */
     947       23636 :                                 file_copy(other_file, file);
     948             : 
     949             :                                 /* revert old counter and use the copy one */
     950       23636 :                                 ++scan->count_copy;
     951             : 
     952       23636 :                                 log_tag("scan:copy:%s:%s:%s:%s\n", other_disk->name, esc_tag(other_file->sub, esc_buffer), disk->name, esc_tag(file->sub, esc_buffer_alt));
     953       23636 :                                 if (is_diff) {
     954          31 :                                         printf("copy %s -> %s\n", fmt_term(other_disk, other_file->sub, esc_buffer), fmt_term(disk, file->sub, esc_buffer_alt));
     955             :                                 }
     956             : 
     957             :                                 /* mark it as reported */
     958       23636 :                                 is_file_reported = 1;
     959             : 
     960             :                                 /* no need to continue the search */
     961       23636 :                                 break;
     962             :                         }
     963             :                 }
     964             :         }
     965             : 
     966             :         /* if not yet reported, do it now */
     967             :         /* we postpone this to avoid to print two times the copied files */
     968       79194 :         if (!is_file_reported) {
     969       55558 :                 if (is_file_already_present) {
     970        4681 :                         ++scan->count_change;
     971             : 
     972       14043 :                         log_tag("scan:update:%s:%s: %" PRIu64 " %" PRIu64 ".%d -> %" PRIu64 " %" PRIu64 ".%d\n", disk->name, esc_tag(sub, esc_buffer),
     973             :                                 (uint64_t)file_already_present_size, (uint64_t)file_already_present_mtime_sec, file_already_present_mtime_nsec,
     974        9362 :                                 (uint64_t)file->size, (uint64_t)file->mtime_sec, file->mtime_nsec
     975             :                         );
     976             : 
     977        4681 :                         if (is_diff) {
     978         118 :                                 printf("update %s\n", fmt_term(disk, sub, esc_buffer));
     979             :                         }
     980             :                 } else {
     981       50877 :                         ++scan->count_insert;
     982             : 
     983       50877 :                         log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
     984       50877 :                         if (is_diff) {
     985       17054 :                                 printf("add %s\n", fmt_term(disk, sub, esc_buffer));
     986             :                         }
     987             :                 }
     988             :         }
     989             : 
     990             :         /* insert the file in the delayed list */
     991       79194 :         scan_file_insert(scan, file);
     992             : }
     993             : 
     994             : /**
     995             :  * Remove the specified dir from the data set.
     996             :  */
     997           2 : static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
     998             : {
     999           2 :         struct snapraid_state* state = scan->state;
    1000           2 :         struct snapraid_disk* disk = scan->disk;
    1001             : 
    1002             :         /* state changed */
    1003           2 :         state->need_write = 1;
    1004             : 
    1005             :         /* remove the file from the dir containers */
    1006           2 :         tommy_hashdyn_remove_existing(&disk->dirset, &dir->nodeset);
    1007           2 :         tommy_list_remove_existing(&disk->dirlist, &dir->nodelist);
    1008             : 
    1009             :         /* deallocate */
    1010           2 :         dir_free(dir);
    1011           2 : }
    1012             : 
    1013             : /**
    1014             :  * Insert the specified dir in the data set.
    1015             :  */
    1016          10 : static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
    1017             : {
    1018          10 :         struct snapraid_state* state = scan->state;
    1019          10 :         struct snapraid_disk* disk = scan->disk;
    1020             : 
    1021             :         /* state changed */
    1022          10 :         state->need_write = 1;
    1023             : 
    1024             :         /* insert the dir in the dir containers */
    1025          10 :         tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
    1026          10 :         tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
    1027          10 : }
    1028             : 
    1029             : /**
    1030             :  * Process a dir.
    1031             :  */
    1032         259 : static void scan_emptydir(struct snapraid_scan* scan, const char* sub)
    1033             : {
    1034         259 :         struct snapraid_disk* disk = scan->disk;
    1035             :         struct snapraid_dir* dir;
    1036             : 
    1037             :         /* check if the dir already exists */
    1038         259 :         dir = tommy_hashdyn_search(&disk->dirset, dir_name_compare, sub, dir_name_hash(sub));
    1039         259 :         if (dir) {
    1040             :                 /* check if multiple files have the same name */
    1041         249 :                 if (dir_flag_has(dir, FILE_IS_PRESENT)) {
    1042             :                         /* LCOV_EXCL_START */
    1043             :                         log_fatal("Internal inconsistency for dir '%s%s'\n", disk->dir, sub);
    1044             :                         os_abort();
    1045             :                         /* LCOV_EXCL_STOP */
    1046             :                 }
    1047             : 
    1048             :                 /* mark as present */
    1049         249 :                 dir_flag_set(dir, FILE_IS_PRESENT);
    1050             : 
    1051             :                 /* nothing more to do */
    1052         508 :                 return;
    1053             :         } else {
    1054             :                 /* and continue to insert it */
    1055             :         }
    1056             : 
    1057             :         /* insert it */
    1058          10 :         dir = dir_alloc(sub);
    1059             : 
    1060             :         /* mark it as present */
    1061          10 :         dir_flag_set(dir, FILE_IS_PRESENT);
    1062             : 
    1063             :         /* insert it in the delayed insert list */
    1064          10 :         tommy_list_insert_tail(&scan->dir_insert_list, &dir->nodelist, dir);
    1065             : }
    1066             : 
    1067             : struct dirent_sorted {
    1068             :         /* node for data structures */
    1069             :         tommy_node node;
    1070             : 
    1071             : #if HAVE_STRUCT_DIRENT_D_INO
    1072             :         uint64_t d_ino; /**< Inode number. */
    1073             : #endif
    1074             : #if HAVE_STRUCT_DIRENT_D_TYPE
    1075             :         uint32_t d_type; /**< File type. */
    1076             : #endif
    1077             : #if HAVE_STRUCT_DIRENT_D_STAT
    1078             :         struct stat d_stat; /**< Stat result. */
    1079             : #endif
    1080             :         char d_name[]; /**< Variable length name. It must be the last field. */
    1081             : };
    1082             : 
    1083             : #if HAVE_STRUCT_DIRENT_D_INO
    1084      103390 : static int dd_ino_compare(const void* void_a, const void* void_b)
    1085             : {
    1086      103390 :         const struct dirent_sorted* a = void_a;
    1087      103390 :         const struct dirent_sorted* b = void_b;
    1088             : 
    1089      103390 :         if (a->d_ino < b->d_ino)
    1090       49438 :                 return -1;
    1091       53952 :         if (a->d_ino > b->d_ino)
    1092       53952 :                 return 1;
    1093             : 
    1094           0 :         return 0;
    1095             : }
    1096             : #endif
    1097             : 
    1098    11526118 : static int dd_name_compare(const void* void_a, const void* void_b)
    1099             : {
    1100    11526118 :         const struct dirent_sorted* a = void_a;
    1101    11526118 :         const struct dirent_sorted* b = void_b;
    1102             : 
    1103    11526118 :         return strcmp(a->d_name, b->d_name);
    1104             : }
    1105             : 
    1106             : /**
    1107             :  * Return the stat info of a dir entry.
    1108             :  */
    1109             : #if HAVE_STRUCT_DIRENT_D_STAT
    1110             : #define DSTAT(file, dd, buf) dstat(dd)
    1111             : struct stat* dstat(struct dirent_sorted* dd)
    1112             : {
    1113             :         return &dd->d_stat;
    1114             : }
    1115             : #else
    1116             : #define DSTAT(file, dd, buf) dstat(file, buf)
    1117     1106210 : struct stat* dstat(const char* file, struct stat* st)
    1118             : {
    1119     1106210 :         if (lstat(file, st) != 0) {
    1120             :                 /* LCOV_EXCL_START */
    1121             :                 log_fatal("Error in stat file/directory '%s'. %s.\n", file, strerror(errno));
    1122             :                 exit(EXIT_FAILURE);
    1123             :                 /* LCOV_EXCL_STOP */
    1124             :         }
    1125     1106210 :         return st;
    1126             : }
    1127             : #endif
    1128             : 
    1129             : /**
    1130             :  * Process a directory.
    1131             :  * Return != 0 if at least one file or link is processed.
    1132             :  */
    1133        1846 : static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const char* dir, const char* sub)
    1134             : {
    1135        1846 :         struct snapraid_state* state = scan->state;
    1136        1846 :         struct snapraid_disk* disk = scan->disk;
    1137        1846 :         int processed = 0;
    1138             :         DIR* d;
    1139             :         tommy_list list;
    1140             :         tommy_node* node;
    1141             : 
    1142        1846 :         tommy_list_init(&list);
    1143             : 
    1144        1846 :         d = opendir(dir);
    1145        1846 :         if (!d) {
    1146             :                 /* LCOV_EXCL_START */
    1147             :                 log_fatal("Error opening directory '%s'. %s.\n", dir, strerror(errno));
    1148             :                 if (level == 0)
    1149             :                         log_fatal("If this is the disk mount point, remember to create it manually\n");
    1150             :                 else
    1151             :                         log_fatal("If it's a permission problem, you can exclude it in the config file with:\n\texclude /%s\n", sub);
    1152             :                 exit(EXIT_FAILURE);
    1153             :                 /* LCOV_EXCL_STOP */
    1154             :         }
    1155             : 
    1156             :         /* read the full directory */
    1157             :         while (1) {
    1158             :                 char path_next[PATH_MAX];
    1159             :                 char sub_next[PATH_MAX];
    1160             :                 struct dirent_sorted* entry;
    1161             :                 const char* name;
    1162             :                 struct dirent* dd;
    1163             :                 size_t name_len;
    1164             : 
    1165             :                 /*
    1166             :                  * Clear errno to differentiate the end of the stream and an error condition
    1167             :                  *
    1168             :                  * From the Linux readdir() manpage:
    1169             :                  * "If the end of the directory stream is reached, NULL is returned and errno is not changed.
    1170             :                  * If an error occurs, NULL is returned and errno is set appropriately."
    1171             :                  */
    1172     1147737 :                 errno = 0;
    1173     1147737 :                 dd = readdir(d);
    1174     1147737 :                 if (dd == 0 && errno != 0) {
    1175             :                         /* LCOV_EXCL_START */
    1176             :                         log_fatal("Error reading directory '%s'. %s.\n", dir, strerror(errno));
    1177             :                         log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub);
    1178             :                         exit(EXIT_FAILURE);
    1179             :                         /* LCOV_EXCL_STOP */
    1180             :                 }
    1181     1147737 :                 if (dd == 0) {
    1182        1846 :                         break; /* finished */
    1183             :                 }
    1184             : 
    1185             :                 /* skip "." and ".." files */
    1186     1145891 :                 name = dd->d_name;
    1187     1145891 :                 if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
    1188        7384 :                         continue;
    1189             : 
    1190     1142199 :                 pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
    1191     1142199 :                 pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
    1192             : 
    1193             :                 /* check for not supported file names */
    1194     1142199 :                 if (name[0] == 0) {
    1195             :                         /* LCOV_EXCL_START */
    1196             :                         log_fatal("Unsupported name '%s' in file '%s'.\n", name, path_next);
    1197             :                         exit(EXIT_FAILURE);
    1198             :                         /* LCOV_EXCL_STOP */
    1199             :                 }
    1200             : 
    1201             :                 /* exclude hidden files even before calling lstat() */
    1202     1142199 :                 if (filter_hidden(state->filter_hidden, dd) != 0) {
    1203           0 :                         msg_verbose("Excluding hidden '%s'\n", path_next);
    1204           0 :                         continue;
    1205             :                 }
    1206             : 
    1207             :                 /* exclude content files even before calling lstat() */
    1208     1142199 :                 if (filter_content(&state->contentlist, path_next) != 0) {
    1209           0 :                         msg_verbose("Excluding content '%s'\n", path_next);
    1210           0 :                         continue;
    1211             :                 }
    1212             : 
    1213     1142199 :                 name_len = strlen(dd->d_name);
    1214     1142199 :                 entry = malloc_nofail(sizeof(struct dirent_sorted) + name_len + 1);
    1215             : 
    1216             :                 /* copy the dir entry */
    1217             : #if HAVE_STRUCT_DIRENT_D_INO
    1218     1142199 :                 entry->d_ino = dd->d_ino;
    1219             : #endif
    1220             : #if HAVE_STRUCT_DIRENT_D_TYPE
    1221     1142199 :                 entry->d_type = dd->d_type;
    1222             : #endif
    1223             : #if HAVE_STRUCT_DIRENT_D_STAT
    1224             :                 /* convert dirent to lstat result */
    1225             :                 dirent_lstat(dd, &entry->d_stat);
    1226             : 
    1227             :                 /* note that at this point the st_mode may be 0 */
    1228             : #endif
    1229     1142199 :                 memcpy(entry->d_name, dd->d_name, name_len + 1);
    1230             : 
    1231             :                 /* insert in the list */
    1232     1142199 :                 tommy_list_insert_tail(&list, &entry->node, entry);
    1233     1145891 :         }
    1234             : 
    1235        1846 :         if (closedir(d) != 0) {
    1236             :                 /* LCOV_EXCL_START */
    1237             :                 log_fatal("Error closing directory '%s'. %s.\n", dir, strerror(errno));
    1238             :                 exit(EXIT_FAILURE);
    1239             :                 /* LCOV_EXCL_STOP */
    1240             :         }
    1241             : 
    1242        1846 :         if (state->opt.force_order == SORT_ALPHA) {
    1243             :                 /* if requested sort alphabetically */
    1244             :                 /* this is mainly done for testing to ensure to always */
    1245             :                 /* process in the same way in different platforms */
    1246        1810 :                 tommy_list_sort(&list, dd_name_compare);
    1247             :         }
    1248             : #if HAVE_STRUCT_DIRENT_D_INO
    1249          36 :         else if (!disk->has_volatile_inodes) {
    1250             :                 /* if inodes are persistent */
    1251             :                 /* sort the list of dir entries by inodes */
    1252          36 :                 tommy_list_sort(&list, dd_ino_compare);
    1253             :         }
    1254             :         /* otherwise just keep the insertion order */
    1255             : #endif
    1256             : 
    1257             :         /* process the sorted dir entries */
    1258        1846 :         node = list;
    1259     1145891 :         while (node != 0) {
    1260             :                 char path_next[PATH_MAX];
    1261             :                 char sub_next[PATH_MAX];
    1262             :                 char out[PATH_MAX];
    1263     1142199 :                 struct snapraid_filter* reason = 0;
    1264     1142199 :                 struct dirent_sorted* dd = node->data;
    1265     1142199 :                 const char* name = dd->d_name;
    1266             :                 struct stat* st;
    1267             :                 int type;
    1268             : #if !HAVE_STRUCT_DIRENT_D_STAT
    1269             :                 struct stat st_buf;
    1270             : #endif
    1271             : 
    1272     1142199 :                 pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
    1273     1142199 :                 pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
    1274             : 
    1275             :                 /* start with an unknown type */
    1276     1142199 :                 type = -1;
    1277     1142199 :                 st = 0;
    1278             : 
    1279             :                 /* if dirent has the type, use it */
    1280             : #if HAVE_STRUCT_DIRENT_D_TYPE
    1281     1142199 :                 switch (dd->d_type) {
    1282           0 :                 case DT_UNKNOWN : break;
    1283     1104939 :                 case DT_REG : type = 0; break;
    1284       35989 :                 case DT_LNK : type = 1; break;
    1285        1271 :                 case DT_DIR : type = 2; break;
    1286           0 :                 default : type = 3; break;
    1287             :                 }
    1288             : #endif
    1289             : 
    1290             :                 /* if type is still unknown */
    1291     1142199 :                 if (type < 0) {
    1292             :                         /* get the type from stat */
    1293           0 :                         st = DSTAT(path_next, dd, &st_buf);
    1294             : 
    1295             : #if HAVE_STRUCT_DIRENT_D_STAT
    1296             :                         /* if the st_mode field is missing, takes care to fill it using normal lstat() */
    1297             :                         /* at now this can happen only in Windows (with HAVE_STRUCT_DIRENT_D_STAT defined), */
    1298             :                         /* because we use a directory reading method that doesn't read info about ReparsePoint. */
    1299             :                         /* Note that here we cannot call here lstat_sync(), because we don't know what kind */
    1300             :                         /* of file is it, and lstat_sync() doesn't always work */
    1301             :                         if (st->st_mode == 0) {
    1302             :                                 if (lstat(path_next, st) != 0) {
    1303             :                                         /* LCOV_EXCL_START */
    1304             :                                         log_fatal("Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
    1305             :                                         exit(EXIT_FAILURE);
    1306             :                                         /* LCOV_EXCL_STOP */
    1307             :                                 }
    1308             :                         }
    1309             : #endif
    1310             : 
    1311           0 :                         if (S_ISREG(st->st_mode))
    1312           0 :                                 type = 0;
    1313           0 :                         else if (S_ISLNK(st->st_mode))
    1314           0 :                                 type = 1;
    1315           0 :                         else if (S_ISDIR(st->st_mode))
    1316           0 :                                 type = 2;
    1317             :                         else
    1318           0 :                                 type = 3;
    1319             :                 }
    1320             : 
    1321     1142199 :                 if (type == 0) { /* REG */
    1322     1104939 :                         if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
    1323             : 
    1324             :                                 /* late stat, if not yet called */
    1325     1104939 :                                 if (!st)
    1326     1104939 :                                         st = DSTAT(path_next, dd, &st_buf);
    1327             : 
    1328             : #if HAVE_LSTAT_SYNC
    1329             :                                 /* if the st_ino field is missing, takes care to fill it using the extended lstat() */
    1330             :                                 /* this can happen only in Windows */
    1331             :                                 if (st->st_ino == 0) {
    1332             :                                         if (lstat_sync(path_next, st, 0) != 0) {
    1333             :                                                 /* LCOV_EXCL_START */
    1334             :                                                 log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
    1335             :                                                 exit(EXIT_FAILURE);
    1336             :                                                 /* LCOV_EXCL_STOP */
    1337             :                                         }
    1338             :                                 }
    1339             : #endif
    1340             : 
    1341     1104939 :                                 scan_file(scan, is_diff, sub_next, st, FILEPHY_UNREAD_OFFSET);
    1342     1104939 :                                 processed = 1;
    1343             :                         } else {
    1344           0 :                                 msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
    1345             :                         }
    1346       37260 :                 } else if (type == 1) { /* LNK */
    1347       35989 :                         if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
    1348             :                                 char subnew[PATH_MAX];
    1349             :                                 int ret;
    1350             : 
    1351       35989 :                                 ret = readlink(path_next, subnew, sizeof(subnew));
    1352       35989 :                                 if (ret >= PATH_MAX) {
    1353             :                                         /* LCOV_EXCL_START */
    1354             :                                         log_fatal("Error in readlink file '%s'. Symlink too long.\n", path_next);
    1355             :                                         exit(EXIT_FAILURE);
    1356             :                                         /* LCOV_EXCL_STOP */
    1357             :                                 }
    1358       35989 :                                 if (ret < 0) {
    1359             :                                         /* LCOV_EXCL_START */
    1360             :                                         log_fatal("Error in readlink file '%s'. %s.\n", path_next, strerror(errno));
    1361             :                                         exit(EXIT_FAILURE);
    1362             :                                         /* LCOV_EXCL_STOP */
    1363             :                                 }
    1364       35989 :                                 if (ret == 0)
    1365           0 :                                         log_fatal("WARNING! Empty symbolic link '%s'.\n", path_next);
    1366             : 
    1367             :                                 /* readlink doesn't put the final 0 */
    1368       35989 :                                 subnew[ret] = 0;
    1369             : 
    1370             :                                 /* process as a symbolic link */
    1371       35989 :                                 scan_link(scan, is_diff, sub_next, subnew, FILE_IS_SYMLINK);
    1372       35989 :                                 processed = 1;
    1373             :                         } else {
    1374           0 :                                 msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
    1375             :                         }
    1376        1271 :                 } else if (type == 2) { /* DIR */
    1377        1271 :                         if (filter_subdir(&state->filterlist, &reason, disk->name, sub_next) == 0) {
    1378             : #ifndef _WIN32
    1379             :                                 /* late stat, if not yet called */
    1380        1271 :                                 if (!st)
    1381        1271 :                                         st = DSTAT(path_next, dd, &st_buf);
    1382             : 
    1383             :                                 /* in Unix don't follow mount points in different devices */
    1384             :                                 /* in Windows we are already skipping them reporting them as special files */
    1385        1271 :                                 if ((uint64_t)st->st_dev != disk->device) {
    1386           0 :                                         log_fatal("WARNING! Ignoring mount point '%s' because it appears to be in a different device\n", path_next);
    1387             :                                 } else
    1388             : #endif
    1389             :                                 {
    1390             :                                         char sub_dir[PATH_MAX];
    1391             : 
    1392             :                                         /* recurse */
    1393        1271 :                                         pathslash(path_next, sizeof(path_next));
    1394        1271 :                                         pathcpy(sub_dir, sizeof(sub_dir), sub_next);
    1395        1271 :                                         pathslash(sub_dir, sizeof(sub_dir));
    1396        1271 :                                         if (scan_dir(scan, level + 1, is_diff, path_next, sub_dir) == 0) {
    1397             :                                                 /* scan the directory as empty dir */
    1398         259 :                                                 scan_emptydir(scan, sub_next);
    1399             :                                         }
    1400             :                                         /* or we processed something internally, or we have added the empty dir */
    1401        1271 :                                         processed = 1;
    1402             :                                 }
    1403             :                         } else {
    1404           0 :                                 msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
    1405             :                         }
    1406             :                 } else {
    1407           0 :                         if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
    1408             :                                 /* late stat, if not yet called */
    1409           0 :                                 if (!st)
    1410           0 :                                         st = DSTAT(path_next, dd, &st_buf);
    1411             : 
    1412           0 :                                 log_fatal("WARNING! Ignoring special '%s' file '%s'\n", stat_desc(st), path_next);
    1413             :                         } else {
    1414           0 :                                 msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
    1415             :                         }
    1416             :                 }
    1417             : 
    1418             :                 /* next entry */
    1419     1142199 :                 node = node->next;
    1420             : 
    1421             :                 /* free the present one */
    1422     1142199 :                 free(dd);
    1423             :         }
    1424             : 
    1425        1846 :         return processed;
    1426             : }
    1427             : 
    1428          96 : static int state_diffscan(struct snapraid_state* state, int is_diff)
    1429             : {
    1430             :         tommy_node* i;
    1431             :         tommy_node* j;
    1432             :         tommy_list scanlist;
    1433             :         int done;
    1434             :         fptr* msg;
    1435             :         struct snapraid_scan total;
    1436             :         int no_difference;
    1437             :         char esc_buffer[ESC_MAX];
    1438             : 
    1439          96 :         tommy_list_init(&scanlist);
    1440             : 
    1441          96 :         if (is_diff)
    1442           8 :                 msg_progress("Comparing...\n");
    1443             : 
    1444             :         /* first scan all the directory and find new and deleted files */
    1445         671 :         for (i = state->disklist; i != 0; i = i->next) {
    1446         575 :                 struct snapraid_disk* disk = i->data;
    1447             :                 struct snapraid_scan* scan;
    1448             :                 tommy_node* node;
    1449             :                 int ret;
    1450             :                 int has_persistent_inode;
    1451             : 
    1452         575 :                 scan = malloc_nofail(sizeof(struct snapraid_scan));
    1453         575 :                 scan->state = state;
    1454         575 :                 scan->disk = disk;
    1455         575 :                 scan->count_equal = 0;
    1456         575 :                 scan->count_move = 0;
    1457         575 :                 scan->count_copy = 0;
    1458         575 :                 scan->count_restore = 0;
    1459         575 :                 scan->count_change = 0;
    1460         575 :                 scan->count_remove = 0;
    1461         575 :                 scan->count_insert = 0;
    1462         575 :                 tommy_list_init(&scan->file_insert_list);
    1463         575 :                 tommy_list_init(&scan->link_insert_list);
    1464         575 :                 tommy_list_init(&scan->dir_insert_list);
    1465             : 
    1466         575 :                 tommy_list_insert_tail(&scanlist, &scan->node, scan);
    1467             : 
    1468         575 :                 if (!is_diff)
    1469         527 :                         msg_progress("Scanning disk %s...\n", disk->name);
    1470             : 
    1471             :                 /* check if the disk supports persistent inodes */
    1472         575 :                 ret = fsinfo(disk->dir, &has_persistent_inode, 0, 0);
    1473         575 :                 if (ret < 0) {
    1474             :                         /* LCOV_EXCL_START */
    1475             :                         log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
    1476             :                         exit(EXIT_FAILURE);
    1477             :                         /* LCOV_EXCL_STOP */
    1478             :                 }
    1479         575 :                 if (!has_persistent_inode) {
    1480           0 :                         disk->has_volatile_inodes = 1;
    1481             :                 }
    1482             : 
    1483             :                 /* if inodes or UUID are not persistent/changed/unsupported */
    1484         575 :                 if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
    1485             :                         /* remove all the inodes from the inode collection */
    1486             :                         /* if they are not persistent, all of them could be changed now */
    1487             :                         /* and we don't want to find false matching ones */
    1488             :                         /* see scan_file() for more details */
    1489          94 :                         node = disk->filelist;
    1490         202 :                         while (node) {
    1491          14 :                                 struct snapraid_file* file = node->data;
    1492             : 
    1493          14 :                                 node = node->next;
    1494             : 
    1495             :                                 /* remove from the inode set */
    1496          14 :                                 tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
    1497             : 
    1498             :                                 /* clear the inode */
    1499          14 :                                 file->inode = 0;
    1500             : 
    1501             :                                 /* mark as missing inode */
    1502          14 :                                 file_flag_set(file, FILE_IS_WITHOUT_INODE);
    1503             :                         }
    1504             :                 }
    1505             : 
    1506         575 :                 scan_dir(scan, 0, is_diff, disk->dir, "");
    1507             :         }
    1508             : 
    1509             :         /* we split the search in two phases because to detect files */
    1510             :         /* moved from one disk to another we have to start deletion */
    1511             :         /* only when all disks have all the new files found */
    1512             : 
    1513             :         /* now process all the new and deleted files */
    1514         671 :         for (i = scanlist; i != 0; i = i->next) {
    1515         575 :                 struct snapraid_scan* scan = i->data;
    1516         575 :                 struct snapraid_disk* disk = scan->disk;
    1517             :                 tommy_node* node;
    1518             :                 unsigned phy_count;
    1519             :                 unsigned phy_dup;
    1520             :                 uint64_t phy_last;
    1521             :                 struct snapraid_file* phy_file_last;
    1522             : 
    1523             :                 /* check for removed files */
    1524         575 :                 node = disk->filelist;
    1525     1033660 :                 while (node) {
    1526     1032510 :                         struct snapraid_file* file = node->data;
    1527             : 
    1528             :                         /* next node */
    1529     1032510 :                         node = node->next;
    1530             : 
    1531             :                         /* remove if not present */
    1532     1032510 :                         if (!file_flag_has(file, FILE_IS_PRESENT)) {
    1533       29647 :                                 ++scan->count_remove;
    1534             : 
    1535       29647 :                                 log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
    1536       29647 :                                 if (is_diff) {
    1537          97 :                                         printf("remove %s\n", fmt_term(disk, file->sub, esc_buffer));
    1538             :                                 }
    1539             : 
    1540       29647 :                                 scan_file_remove(scan, file);
    1541             :                         }
    1542             :                 }
    1543             : 
    1544             :                 /* check for removed links */
    1545         575 :                 node = disk->linklist;
    1546       35833 :                 while (node) {
    1547       34683 :                         struct snapraid_link* slink = node->data;
    1548             : 
    1549             :                         /* next node */
    1550       34683 :                         node = node->next;
    1551             : 
    1552             :                         /* remove if not present */
    1553       34683 :                         if (!link_flag_has(slink, FILE_IS_PRESENT)) {
    1554         956 :                                 ++scan->count_remove;
    1555             : 
    1556         956 :                                 log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
    1557         956 :                                 if (is_diff) {
    1558          10 :                                         printf("remove %s\n", fmt_term(disk, slink->sub, esc_buffer));
    1559             :                                 }
    1560             : 
    1561         956 :                                 scan_link_remove(scan, slink);
    1562             :                         }
    1563             :                 }
    1564             : 
    1565             :                 /* check for removed dirs */
    1566         575 :                 node = disk->dirlist;
    1567        1401 :                 while (node) {
    1568         251 :                         struct snapraid_dir* dir = node->data;
    1569             : 
    1570             :                         /* next node */
    1571         251 :                         node = node->next;
    1572             : 
    1573             :                         /* remove if not present */
    1574         251 :                         if (!dir_flag_has(dir, FILE_IS_PRESENT)) {
    1575           2 :                                 scan_emptydir_remove(scan, dir);
    1576             :                         }
    1577             :                 }
    1578             : 
    1579             :                 /* sort the files before inserting them */
    1580             :                 /* we use a stable sort to ensure that if the reported physical offset/inode */
    1581             :                 /* are always 0, we keep at least the directory order */
    1582         575 :                 switch (state->opt.force_order) {
    1583             :                 case SORT_PHYSICAL :
    1584          12 :                         tommy_list_sort(&scan->file_insert_list, file_physical_compare);
    1585          12 :                         break;
    1586             :                 case SORT_INODE :
    1587           0 :                         tommy_list_sort(&scan->file_insert_list, file_inode_compare);
    1588           0 :                         break;
    1589             :                 case SORT_ALPHA :
    1590         563 :                         tommy_list_sort(&scan->file_insert_list, file_path_compare);
    1591         563 :                         break;
    1592             :                 case SORT_DIR :
    1593             :                         /* already in order */
    1594           0 :                         break;
    1595             :                 }
    1596             : 
    1597             :                 /* insert all the new files, we insert them only after the deletion */
    1598             :                 /* to reuse the just freed space */
    1599             :                 /* also check if the physical offset reported are fakes or not */
    1600         575 :                 node = scan->file_insert_list;
    1601         575 :                 phy_count = 0;
    1602         575 :                 phy_dup = 0;
    1603         575 :                 phy_last = FILEPHY_UNREAD_OFFSET;
    1604         575 :                 phy_file_last = 0;
    1605      103031 :                 while (node) {
    1606      101881 :                         struct snapraid_file* file = node->data;
    1607             : 
    1608             :                         /* if the file is not empty, count duplicate physical offsets */
    1609      101881 :                         if (state->opt.force_order == SORT_PHYSICAL && file->size != 0) {
    1610       11418 :                                 if (phy_file_last != 0 && file->physical == phy_last
    1611             :                                         /* files without offset are expected to have duplicates */
    1612           0 :                                         && phy_last != FILEPHY_WITHOUT_OFFSET
    1613             :                                 ) {
    1614             :                                         /* if verbose, print the list of duplicates real offsets */
    1615             :                                         /* other cases are for offsets not supported, so we don't need to report them file by file */
    1616           0 :                                         if (phy_last >= FILEPHY_REAL_OFFSET) {
    1617           0 :                                                 log_fatal("WARNING! Files '%s%s' and '%s%s' have the same physical offset %" PRId64 ".\n", disk->dir, phy_file_last->sub, disk->dir, file->sub, phy_last);
    1618             :                                         }
    1619           0 :                                         ++phy_dup;
    1620             :                                 }
    1621       11418 :                                 phy_file_last = file;
    1622       11418 :                                 phy_last = file->physical;
    1623       11418 :                                 ++phy_count;
    1624             :                         }
    1625             : 
    1626             :                         /* next node */
    1627      101881 :                         node = node->next;
    1628             : 
    1629             :                         /* insert in the parity */
    1630      101881 :                         scan_file_allocate(scan, file);
    1631             :                 }
    1632             : 
    1633             :                 /* mark the disk without reliable physical offset if it has duplicates */
    1634             :                 /* here it should never happen because we already sorted out hardlinks */
    1635         575 :                 if (state->opt.force_order == SORT_PHYSICAL && phy_dup > 0) {
    1636           0 :                         disk->has_unreliable_physical = 1;
    1637             :                 }
    1638             : 
    1639             :                 /* insert all the new links */
    1640         575 :                 node = scan->link_insert_list;
    1641        3607 :                 while (node) {
    1642        2457 :                         struct snapraid_link* slink = node->data;
    1643             : 
    1644             :                         /* next node */
    1645        2457 :                         node = node->next;
    1646             : 
    1647             :                         /* insert it */
    1648        2457 :                         scan_link_insert(scan, slink);
    1649             :                 }
    1650             : 
    1651             :                 /* insert all the new dirs */
    1652         575 :                 node = scan->dir_insert_list;
    1653        1160 :                 while (node) {
    1654          10 :                         struct snapraid_dir* dir = node->data;
    1655             : 
    1656             :                         /* next node */
    1657          10 :                         node = node->next;
    1658             : 
    1659             :                         /* insert it */
    1660          10 :                         scan_emptydir_insert(scan, dir);
    1661             :                 }
    1662             :         }
    1663             : 
    1664             :         /* check for disks where all the previously existing files where removed */
    1665          96 :         if (!state->opt.force_empty) {
    1666          90 :                 int all_missing = 0;
    1667          90 :                 int all_rewritten = 0;
    1668          90 :                 done = 0;
    1669         629 :                 for (i = state->disklist, j = scanlist; i != 0; i = i->next, j = j->next) {
    1670         539 :                         struct snapraid_disk* disk = i->data;
    1671         539 :                         struct snapraid_scan* scan = j->data;
    1672             : 
    1673         539 :                         if (scan->count_equal == 0
    1674          74 :                                 && scan->count_move == 0
    1675          74 :                                 && scan->count_restore == 0
    1676          74 :                                 && (scan->count_remove != 0 || scan->count_change != 0)
    1677             :                         ) {
    1678           1 :                                 if (!done) {
    1679           1 :                                         done = 1;
    1680           1 :                                         log_fatal("WARNING! All the files previously present in disk '%s' at dir '%s'", disk->name, disk->dir);
    1681             :                                 } else {
    1682           0 :                                         log_fatal(", disk '%s' at dir '%s'", disk->name, disk->dir);
    1683             :                                 }
    1684             : 
    1685             :                                 /* detect the special condition of all files missing */
    1686           1 :                                 if (scan->count_change == 0)
    1687           1 :                                         all_missing = 1;
    1688             : 
    1689             :                                 /* detect the special condition of all files rewritten */
    1690           1 :                                 if (scan->count_remove == 0)
    1691           0 :                                         all_rewritten = 1;
    1692             :                         }
    1693             :                 }
    1694          90 :                 if (done) {
    1695           1 :                         log_fatal("\nare now missing or rewritten!\n");
    1696           1 :                         if (all_rewritten) {
    1697           0 :                                 log_fatal("This could happen when restoring a disk with a backup\n");
    1698           0 :                                 log_fatal("program that is not setting correctly the timestamps.\n");
    1699             :                         }
    1700           1 :                         if (all_missing) {
    1701           1 :                                 log_fatal("This could happen when some disks are not mounted\n");
    1702           1 :                                 log_fatal("in the expected directory.\n");
    1703             :                         }
    1704           1 :                         if (!is_diff) {
    1705           1 :                                 log_fatal("If you want to '%s' anyway, use 'snapraid --force-empty %s'.\n", state->command, state->command);
    1706           1 :                                 exit(EXIT_FAILURE);
    1707             :                         }
    1708             :                 }
    1709             :         }
    1710             : 
    1711             :         /* check for disks without the physical offset support */
    1712          95 :         if (state->opt.force_order == SORT_PHYSICAL) {
    1713           2 :                 done = 0;
    1714          14 :                 for (i = state->disklist; i != 0; i = i->next) {
    1715          12 :                         struct snapraid_disk* disk = i->data;
    1716             : 
    1717          12 :                         if (disk->has_unreliable_physical) {
    1718           0 :                                 if (!done) {
    1719           0 :                                         done = 1;
    1720           0 :                                         log_fatal("WARNING! Physical offsets not supported for disk '%s'", disk->name);
    1721             :                                 } else {
    1722           0 :                                         log_fatal(", '%s'", disk->name);
    1723             :                                 }
    1724             :                         }
    1725             :                 }
    1726           2 :                 if (done) {
    1727           0 :                         log_fatal(". Files order won't be optimal.\n");
    1728             :                 }
    1729             :         }
    1730             : 
    1731             :         /* check for disks without persistent inodes */
    1732          95 :         done = 0;
    1733         664 :         for (i = state->disklist; i != 0; i = i->next) {
    1734         569 :                 struct snapraid_disk* disk = i->data;
    1735             : 
    1736         569 :                 if (disk->has_volatile_inodes) {
    1737           0 :                         if (!done) {
    1738           0 :                                 done = 1;
    1739           0 :                                 log_fatal("WARNING! Inodes are not persistent for disks: '%s'", disk->name);
    1740             :                         } else {
    1741           0 :                                 log_fatal(", '%s'", disk->name);
    1742             :                         }
    1743             :                 }
    1744             :         }
    1745          95 :         if (done) {
    1746           0 :                 log_fatal(". Not using inodes to detect move operations.\n");
    1747             :         }
    1748             : 
    1749             :         /* check for disks with changed UUID */
    1750          95 :         done = 0;
    1751         664 :         for (i = state->disklist; i != 0; i = i->next) {
    1752         569 :                 struct snapraid_disk* disk = i->data;
    1753             : 
    1754             :                 /* don't print the message if the UUID changed because before */
    1755             :                 /* it was no set. */
    1756             :                 /* this is the normal condition for an empty disk because it */
    1757             :                 /* isn't stored */
    1758         569 :                 if (disk->has_different_uuid && !disk->had_empty_uuid) {
    1759           4 :                         if (!done) {
    1760           2 :                                 done = 1;
    1761           2 :                                 log_fatal("WARNING! UUID is changed for disks: '%s'", disk->name);
    1762             :                         } else {
    1763           2 :                                 log_fatal(", '%s'", disk->name);
    1764             :                         }
    1765             :                 }
    1766             :         }
    1767          95 :         if (done) {
    1768           2 :                 log_fatal(". Not using inodes to detect move operations.\n");
    1769             :         }
    1770             : 
    1771             :         /* check for disks with unsupported UUID */
    1772          95 :         done = 0;
    1773         664 :         for (i = state->disklist; i != 0; i = i->next) {
    1774         569 :                 struct snapraid_disk* disk = i->data;
    1775             : 
    1776         569 :                 if (disk->has_unsupported_uuid) {
    1777           0 :                         if (!done) {
    1778           0 :                                 done = 1;
    1779           0 :                                 log_fatal("WARNING! UUID is unsupported for disks: '%s'", disk->name);
    1780             :                         } else {
    1781           0 :                                 log_fatal(", '%s'", disk->name);
    1782             :                         }
    1783             :                 }
    1784             :         }
    1785          95 :         if (done) {
    1786           0 :                 log_fatal(". Not using inodes to detect move operations.\n");
    1787             : #if defined(_linux) && !HAVE_BLKID
    1788             :                 log_fatal("The 'blkid' library is not linked in SnapRAID!\n");
    1789             :                 log_fatal("Try rebuilding it after installing the libblkid-dev or libblkid-devel package.\n");
    1790             : #endif
    1791             :         }
    1792             : 
    1793          95 :         total.count_equal = 0;
    1794          95 :         total.count_move = 0;
    1795          95 :         total.count_copy = 0;
    1796          95 :         total.count_restore = 0;
    1797          95 :         total.count_change = 0;
    1798          95 :         total.count_remove = 0;
    1799          95 :         total.count_insert = 0;
    1800             : 
    1801         664 :         for (i = scanlist; i != 0; i = i->next) {
    1802         569 :                 struct snapraid_scan* scan = i->data;
    1803         569 :                 total.count_equal += scan->count_equal;
    1804         569 :                 total.count_move += scan->count_move;
    1805         569 :                 total.count_copy += scan->count_copy;
    1806         569 :                 total.count_restore += scan->count_restore;
    1807         569 :                 total.count_change += scan->count_change;
    1808         569 :                 total.count_remove += scan->count_remove;
    1809         569 :                 total.count_insert += scan->count_insert;
    1810             :         }
    1811             : 
    1812          95 :         if (is_diff) {
    1813           8 :                 msg_status("\n");
    1814           8 :                 msg = msg_status;
    1815             :         } else {
    1816          87 :                 msg = msg_verbose;
    1817             :         }
    1818             : 
    1819          95 :         msg("%8u equal\n", total.count_equal);
    1820          95 :         msg("%8u added\n", total.count_insert);
    1821          95 :         msg("%8u removed\n", total.count_remove);
    1822          95 :         msg("%8u updated\n", total.count_change);
    1823          95 :         msg("%8u moved\n", total.count_move);
    1824          95 :         msg("%8u copied\n", total.count_copy);
    1825          95 :         msg("%8u restored\n", total.count_restore);
    1826             : 
    1827          95 :         log_tag("summary:equal:%u\n", total.count_equal);
    1828          95 :         log_tag("summary:added:%u\n", total.count_insert);
    1829          95 :         log_tag("summary:removed:%u\n", total.count_remove);
    1830          95 :         log_tag("summary:updated:%u\n", total.count_change);
    1831          95 :         log_tag("summary:moved:%u\n", total.count_move);
    1832          95 :         log_tag("summary:copied:%u\n", total.count_copy);
    1833          95 :         log_tag("summary:restored:%u\n", total.count_restore);
    1834             : 
    1835         284 :         no_difference = !total.count_move && !total.count_copy && !total.count_restore
    1836         159 :                 && !total.count_change && !total.count_remove && !total.count_insert;
    1837             : 
    1838          95 :         if (is_diff) {
    1839           8 :                 if (no_difference) {
    1840           2 :                         msg_status("No differences\n");
    1841             :                 } else {
    1842           6 :                         msg_status("There are differences!\n");
    1843             :                 }
    1844             :         }
    1845             : 
    1846          95 :         if (no_difference) {
    1847          20 :                 log_tag("summary:exit:equal\n");
    1848             :         } else {
    1849          75 :                 log_tag("summary:exit:diff\n");
    1850             :         }
    1851          95 :         log_flush();
    1852             : 
    1853          95 :         tommy_list_foreach(&scanlist, (tommy_foreach_func*)free);
    1854             : 
    1855             :         /* check the file-system on all disks */
    1856          95 :         state_fscheck(state, "after scan");
    1857             : 
    1858          95 :         if (is_diff) {
    1859             :                 /* check for file difference */
    1860           8 :                 if (!no_difference)
    1861           6 :                         return 1;
    1862             : 
    1863             :                 /* check also for incomplete "sync" */
    1864           2 :                 if (parity_is_invalid(state))
    1865           1 :                         return 1;
    1866             :         }
    1867             : 
    1868          88 :         return 0;
    1869             : }
    1870             : 
    1871           8 : int state_diff(struct snapraid_state* state)
    1872             : {
    1873           8 :         return state_diffscan(state, 1);
    1874             : }
    1875             : 
    1876          88 : void state_scan(struct snapraid_state* state)
    1877             : {
    1878          88 :         (void)state_diffscan(state, 0); /* ignore return value */
    1879          87 : }
    1880             : 

Generated by: LCOV version 1.10