Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "support.h"
21 : #include "elem.h"
22 : #include "state.h"
23 : #include "parity.h"
24 :
25 : struct snapraid_scan {
26 : struct snapraid_state* state; /**< State used. */
27 : struct snapraid_disk* disk; /**< Disk used. */
28 :
29 : /**
30 : * Counters of changes.
31 : */
32 : unsigned count_equal; /**< Files equal. */
33 : unsigned count_move; /**< Files with a different name, but equal inode, size and timestamp in the same disk. */
34 : unsigned count_restore; /**< Files with equal name, size and timestamp, but different inode. */
35 : unsigned count_change; /**< Files with same name, but different size and/or timestamp. */
36 : unsigned count_copy; /**< Files new, with same name size and timestamp of a file in a different disk. */
37 : unsigned count_insert; /**< Files new. */
38 : unsigned count_remove; /**< Files removed. */
39 :
40 : tommy_list file_insert_list; /**< Files to insert. */
41 : tommy_list link_insert_list; /**< Links to insert. */
42 : tommy_list dir_insert_list; /**< Dirs to insert. */
43 :
44 : /* nodes for data structures */
45 : tommy_node node;
46 : };
47 :
48 : /**
49 : * Removes the specified link from the data set.
50 : */
51 956 : static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* link)
52 : {
53 956 : struct snapraid_state* state = scan->state;
54 956 : struct snapraid_disk* disk = scan->disk;
55 :
56 : /* state changed */
57 956 : state->need_write = 1;
58 :
59 : /* remove the file from the link containers */
60 956 : tommy_hashdyn_remove_existing(&disk->linkset, &link->nodeset);
61 956 : tommy_list_remove_existing(&disk->linklist, &link->nodelist);
62 :
63 : /* deallocate */
64 956 : link_free(link);
65 956 : }
66 :
67 : /**
68 : * Inserts the specified link in the data set.
69 : */
70 2457 : static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* link)
71 : {
72 2457 : struct snapraid_state* state = scan->state;
73 2457 : struct snapraid_disk* disk = scan->disk;
74 :
75 : /* state changed */
76 2457 : state->need_write = 1;
77 :
78 : /* insert the link in the link containers */
79 2457 : tommy_hashdyn_insert(&disk->linkset, &link->nodeset, link, link_name_hash(link->sub));
80 2457 : tommy_list_insert_tail(&disk->linklist, &link->nodelist, link);
81 2457 : }
82 :
83 : /**
84 : * Processes a symbolic link.
85 : */
86 31866 : static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub, const char* linkto, unsigned link_flag)
87 : {
88 31866 : struct snapraid_state* state = scan->state;
89 31866 : struct snapraid_disk* disk = scan->disk;
90 : struct snapraid_link* link;
91 :
92 : /* check if the link already exists */
93 31866 : link = tommy_hashdyn_search(&disk->linkset, link_name_compare_to_arg, sub, link_name_hash(sub));
94 31866 : if (link) {
95 : /* check if multiple files have the same name */
96 29409 : if (link_flag_has(link, FILE_IS_PRESENT)) {
97 : /* LCOV_EXCL_START */
98 : log_fatal("Internal inconsistency for link '%s%s'\n", disk->dir, sub);
99 : exit(EXIT_FAILURE);
100 : /* LCOV_EXCL_STOP */
101 : }
102 :
103 : /* mark as present */
104 29409 : link_flag_set(link, FILE_IS_PRESENT);
105 :
106 : /* check if the link is not changed and it's of the same kind */
107 29409 : if (strcmp(link->linkto, linkto) == 0 && link_flag == link_flag_get(link, FILE_IS_LINK_MASK)) {
108 : /* it's equal */
109 29409 : ++scan->count_equal;
110 :
111 58818 : if (state->opt.gui) {
112 829 : log_tag("scan:equal:%s:%s\n", disk->name, esc(link->sub));
113 : }
114 : } else {
115 : /* it's an update */
116 :
117 : /* we have to save the linkto/type */
118 0 : state->need_write = 1;
119 :
120 0 : ++scan->count_change;
121 :
122 0 : log_tag("scan:update:%s:%s\n", disk->name, esc(link->sub));
123 0 : if (is_diff) {
124 0 : printf("update %s%s\n", disk->dir, link->sub);
125 : }
126 :
127 : /* update it */
128 0 : free(link->linkto);
129 0 : link->linkto = strdup_nofail(linkto);
130 0 : link_flag_let(link, link_flag, FILE_IS_LINK_MASK);
131 : }
132 :
133 : /* nothing more to do */
134 61275 : return;
135 : } else {
136 : /* create the new link */
137 2457 : ++scan->count_insert;
138 :
139 2457 : log_tag("scan:add:%s:%s\n", disk->name, esc(sub));
140 2457 : if (is_diff) {
141 598 : printf("add %s%s\n", disk->dir, sub);
142 : }
143 :
144 : /* and continue to insert it */
145 : }
146 :
147 : /* insert it */
148 2457 : link = link_alloc(sub, linkto, link_flag);
149 :
150 : /* mark it as present */
151 2457 : link_flag_set(link, FILE_IS_PRESENT);
152 :
153 : /* insert it in the delayed insert list */
154 2457 : tommy_list_insert_tail(&scan->link_insert_list, &link->nodelist, link);
155 : }
156 :
157 : /**
158 : * Removes the specified file from the parity.
159 : */
160 55207 : static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
161 : {
162 55207 : struct snapraid_state* state = scan->state;
163 55207 : struct snapraid_disk* disk = scan->disk;
164 :
165 : block_off_t i;
166 :
167 : /* state changed */
168 55207 : state->need_write = 1;
169 :
170 : /* free all the blocks of the file */
171 191141 : for (i = 0; i < file->blockmax; ++i) {
172 135934 : struct snapraid_block* block = &file->blockvec[i];
173 135934 : block_off_t block_pos = block->parity_pos;
174 : unsigned block_state;
175 : struct snapraid_deleted* deleted;
176 :
177 : /* adjust the first free position */
178 : /* note that doing all the deletions before alllocations, */
179 : /* first_free_block is always 0 and the "if" is never triggered */
180 : /* but we keep this code anyway for completeness. */
181 135934 : if (disk->first_free_block > block_pos)
182 0 : disk->first_free_block = block_pos;
183 :
184 : /* allocated a new deleted block from the block we are going to delete */
185 135934 : deleted = deleted_dup(block);
186 :
187 : /* in case we scan after an aborted sync, */
188 : /* we could get also intermediate states */
189 135934 : block_state = block_state_get(block);
190 135934 : switch (block_state) {
191 : case BLOCK_STATE_BLK :
192 : /* we keep the hash making it an "old" hash, because the parity is still containing data for it */
193 31139 : break;
194 : case BLOCK_STATE_CHG :
195 : /* if we have not already cleared the past hash */
196 57633 : if (!state->clear_past_hash) {
197 : /* in these cases we don't know if the old state is still the one */
198 : /* stored inside the parity, because after an aborted sync, the parity */
199 : /* may be or may be not have been updated with the data that it's now */
200 : /* deleted. Then we reset the hash to a bogus value. */
201 : /* For example: */
202 : /* - One file is added */
203 : /* - Sync aborted after updating the parity to the new state, */
204 : /* but without saving the content file representing this new state. */
205 : /* - File is now deleted after the aborted sync */
206 : /* - Sync again, deleting the blocks (exactly here) */
207 : /* with the hash of CHG block not represeting the real parity state */
208 0 : hash_invalid_set(deleted->block.hash);
209 : }
210 57633 : break;
211 : case BLOCK_STATE_REP :
212 : /* we just don't know the old hash, and then we set it to invalid */
213 47162 : hash_invalid_set(deleted->block.hash);
214 47162 : break;
215 : default :
216 : /* LCOV_EXCL_START */
217 : log_fatal("Internal inconsistency in deallocating for block %u state %u\n", block->parity_pos, block_state);
218 : exit(EXIT_FAILURE);
219 : /* LCOV_EXCL_STOP */
220 : }
221 :
222 : /* insert it in the list of deleted blocks */
223 135934 : tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
224 :
225 : /* set the deleted block in the block array */
226 135934 : tommy_arrayblk_set(&disk->blockarr, block_pos, &deleted->block);
227 : }
228 55207 : }
229 :
230 : /**
231 : * Checks if a file is completely formed of blocks with invalid parity,
232 : * and no rehash is tagged, and if it has at least one block.
233 : */
234 897263 : static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_file* file)
235 : {
236 : block_off_t i;
237 :
238 : /* with no block, it never has an invalid parity */
239 897263 : if (file->blockmax == 0)
240 1187 : return 0;
241 :
242 : /* check all blocks */
243 951882 : for (i = 0; i < file->blockmax; ++i) {
244 929217 : struct snapraid_block* block = &file->blockvec[i];
245 : snapraid_info info;
246 :
247 : /* exclude blocks with parity */
248 929217 : if (!block_has_invalid_parity(block))
249 873411 : return 0;
250 :
251 : /* get block specific info */
252 55806 : info = info_get(&state->infoarr, block->parity_pos);
253 :
254 : /* if rehash fails */
255 55806 : if (info_get_rehash(info))
256 0 : return 0;
257 : }
258 :
259 22665 : return 1;
260 : }
261 :
262 : /**
263 : * Checks if a file is completely formed of blocks with an updated hash,
264 : * and no rehash is tagged, and if it has at least one block.
265 : */
266 23658 : static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_file* file)
267 : {
268 : block_off_t i;
269 :
270 : /* with no block, it never has a hash */
271 23658 : if (file->blockmax == 0)
272 27 : return 0;
273 :
274 : /* check all blocks */
275 42139 : for (i = 0; i < file->blockmax; ++i) {
276 35994 : struct snapraid_block* block = &file->blockvec[i];
277 : snapraid_info info;
278 :
279 : /* exclude blocks without hash */
280 35994 : if (!block_has_updated_hash(block))
281 1702 : return 0;
282 :
283 : /* get block specific info */
284 34292 : info = info_get(&state->infoarr, block->parity_pos);
285 :
286 : /* exclude blocks needing a rehash */
287 34292 : if (info_get_rehash(info))
288 15784 : return 0;
289 : }
290 :
291 6145 : return 1;
292 : }
293 :
294 100042 : static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid_file* file)
295 : {
296 100042 : struct snapraid_state* state = scan->state;
297 100042 : struct snapraid_disk* disk = scan->disk;
298 :
299 : /* if we sort for physical offsets we have to read them for new files */
300 100042 : if (state->opt.force_order == SORT_PHYSICAL
301 11430 : && file->physical == FILEPHY_UNREAD_OFFSET
302 : ) {
303 : char path_next[PATH_MAX];
304 :
305 11430 : pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, file->sub);
306 :
307 11430 : if (filephy(path_next, file->size, &file->physical) != 0) {
308 : /* LCOV_EXCL_START */
309 : log_fatal("Error in getting the physical offset of file '%s'. %s.\n", path_next, strerror(errno));
310 : exit(EXIT_FAILURE);
311 : /* LCOV_EXCL_STOP */
312 : }
313 : }
314 :
315 100042 : tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
316 100042 : }
317 :
318 : /**
319 : * Refresh the file info.
320 : *
321 : * This is needed by Windows as the normal way to list directories may report not
322 : * updated info. Only the GetFileInformationByHandle() func, called file-by-file,
323 : * really ensures to return synced info.
324 : *
325 : * If this happens, we read also the physical offset, to avoid to read it later.
326 : */
327 77377 : static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struct stat* st, uint64_t* physical)
328 : {
329 : #if HAVE_LSTAT_SYNC
330 : struct snapraid_state* state = scan->state;
331 : struct snapraid_disk* disk = scan->disk;
332 :
333 : /* if the st_sync is not set, ensure to get synced info */
334 : if (st->st_sync == 0) {
335 : char path_next[PATH_MAX];
336 : struct stat synced_st;
337 :
338 : pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, sub);
339 :
340 : /* if we sort for physical offsets we have to read them for new files */
341 : if (state->opt.force_order == SORT_PHYSICAL
342 : && *physical == FILEPHY_UNREAD_OFFSET
343 : ) {
344 : /* do nothing, leave the pointer to read the physical offset */
345 : } else {
346 : physical = 0; /* set the pointer to 0 to read nothing */
347 : }
348 :
349 : if (lstat_sync(path_next, &synced_st, physical) != 0) {
350 : /* LCOV_EXCL_START */
351 : log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
352 : exit(EXIT_FAILURE);
353 : /* LCOV_EXCL_STOP */
354 : }
355 :
356 : if (st->st_mtime != synced_st.st_mtime
357 : || st->st_mtimensec != synced_st.st_mtimensec
358 : ) {
359 : #ifndef _WIN32
360 : /*
361 : * In Windows having different metadata is expected with open files
362 : * because the metadata in the directory is updated only when the file
363 : * is closed.
364 : *
365 : * See also:
366 : * Why is the file size reported incorrectly for files that are still being written to?
367 : * http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx
368 : */
369 : log_fatal("WARNING! Detected uncached time change for file '%s'\n", sub);
370 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
371 : #endif
372 : st->st_mtime = synced_st.st_mtime;
373 : st->st_mtimensec = synced_st.st_mtimensec;
374 : }
375 :
376 : if (st->st_size != synced_st.st_size) {
377 : #ifndef _WIN32
378 : log_fatal("WARNING! Detected uncached size change for file '%s'\n", sub);
379 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
380 : #endif
381 : st->st_size = synced_st.st_size;
382 : }
383 :
384 : if (st->st_ino != synced_st.st_ino) {
385 : log_fatal("DANGER! Detected uncached inode change for file '%s'\n", sub);
386 : log_fatal("Please ensure to run SnapRAID without other processes running.\n");
387 : /* at this point, it's too late to change inode */
388 : /* and having inconsistent inodes may result to internal failures */
389 : /* so, it's better to abort */
390 : exit(EXIT_FAILURE);
391 : }
392 : }
393 : #else
394 : (void)scan;
395 : (void)sub;
396 : (void)st;
397 : (void)physical;
398 : #endif
399 77377 : }
400 :
401 : /**
402 : * Keeps the file as it's (or with only a name/inode modification).
403 : *
404 : * If the file is kept, nothing has to be done.
405 : *
406 : * But if a file contains only blocks with invalid parity, it's reallocated to ensure
407 : * to always minimize the space used in the parity.
408 : *
409 : * This could happen after a failed sync, when some other files are deleted,
410 : * and then new ones can be moved backward to fill the hole created.
411 : */
412 897263 : static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* file)
413 : {
414 897263 : struct snapraid_disk* disk = scan->disk;
415 :
416 : /* if the file is full invalid, schedule a reinsert at later stage */
417 897263 : if (file_is_full_invalid_parity_and_stable(scan->state, file)) {
418 : /* deallocate the file from the parity */
419 22665 : scan_file_deallocate(scan, file);
420 :
421 : /* remove the file from the list */
422 22665 : tommy_list_remove_existing(&disk->filelist, &file->nodelist);
423 :
424 : /* insert the file in the delayed block allocation */
425 22665 : scan_file_delayed_insert(scan, file);
426 : }
427 897263 : }
428 :
429 : /**
430 : * Removes the specified file from the data set.
431 : */
432 32542 : static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
433 : {
434 32542 : struct snapraid_disk* disk = scan->disk;
435 :
436 : /* deallocate the file from the parity */
437 32542 : scan_file_deallocate(scan, file);
438 :
439 : /* remove the file from the file containers */
440 32542 : if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
441 32501 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
442 : }
443 32542 : tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
444 32542 : tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
445 32542 : tommy_list_remove_existing(&disk->filelist, &file->nodelist);
446 :
447 : /* deallocate */
448 32542 : file_free(file);
449 32542 : }
450 :
451 : /**
452 : * Inserts the specified file in the data set.
453 : */
454 100042 : static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
455 : {
456 100042 : struct snapraid_state* state = scan->state;
457 100042 : struct snapraid_disk* disk = scan->disk;
458 : block_off_t i;
459 : block_off_t block_max;
460 : block_off_t block_pos;
461 :
462 : /* state changed */
463 100042 : state->need_write = 1;
464 :
465 : /* allocate the blocks of the file */
466 100042 : block_pos = disk->first_free_block;
467 100042 : block_max = tommy_arrayblk_size(&disk->blockarr);
468 346361 : for (i = 0; i < file->blockmax; ++i) {
469 : snapraid_info info;
470 246319 : struct snapraid_block* block = &file->blockvec[i];
471 :
472 : /* increment the position until the first really free block */
473 994796 : while (block_pos < block_max && block_has_file(tommy_arrayblk_get(&disk->blockarr, block_pos)))
474 502158 : ++block_pos;
475 :
476 : /* if not found, allocate a new one */
477 246319 : if (block_pos == block_max) {
478 155209 : ++block_max;
479 155209 : tommy_arrayblk_grow(&disk->blockarr, block_max);
480 : }
481 :
482 : /* set the position */
483 246319 : block->parity_pos = block_pos;
484 :
485 : /* get block specific info */
486 246319 : info = info_get(&state->infoarr, block_pos);
487 :
488 : /* if the file block already has an updated hash without rehash */
489 246319 : if (block_has_updated_hash(block) && !info_get_rehash(info)) {
490 : /* the only possible case is for REP blocks */
491 39390 : assert(block_state_get(block) == BLOCK_STATE_REP);
492 :
493 : /* convert to a REP block */
494 39390 : block_state_set(&file->blockvec[i], BLOCK_STATE_REP);
495 :
496 : /* and keep the hash as it's */
497 : } else {
498 : struct snapraid_block* over_block;
499 : unsigned block_state;
500 :
501 : /* convert to a CHG block */
502 206929 : block_state_set(block, BLOCK_STATE_CHG);
503 :
504 : /* block to overwrite */
505 206929 : over_block = tommy_arrayblk_get(&disk->blockarr, block_pos);
506 :
507 : /* state of the block we are going to overwrite */
508 206929 : block_state = block_state_get(over_block);
509 :
510 : /* if the block is an empty one */
511 206929 : if (block_state == BLOCK_STATE_EMPTY) {
512 : /* the block was empty and filled with zeros */
513 : /* set the hash to the special ZERO value */
514 153240 : hash_zero_set(block->hash);
515 : } else {
516 : /* otherwise it's a DELETED one */
517 53689 : assert(block_state == BLOCK_STATE_DELETED);
518 :
519 : /* copy the past hash of the block */
520 53689 : memcpy(block->hash, over_block->hash, HASH_SIZE);
521 :
522 : /* if we have not already cleared the past hash */
523 53689 : if (!state->clear_past_hash) {
524 : /* in this case we don't know if the old state is still the one */
525 : /* stored inside the parity, because after an aborted sync, the parity */
526 : /* may be or may be not have been updated with the new data */
527 : /* Then we reset the hash to a bogus value */
528 : /* For example: */
529 : /* - One file is deleted */
530 : /* - Sync aborted after, updating the parity to the new state, */
531 : /* but without saving the content file representing this new state. */
532 : /* - Another file is added again (exactly here) */
533 : /* with the hash of DELETED block not represeting the real parity state */
534 380 : hash_invalid_set(block->hash);
535 : }
536 : }
537 : }
538 :
539 : /* store in the disk map, after invalidating all the other blocks */
540 246319 : tommy_arrayblk_set(&disk->blockarr, block_pos, block);
541 :
542 : /* set the new free position */
543 246319 : disk->first_free_block = block_pos + 1;
544 : }
545 :
546 : /* note that the file is already added in the file hashtables */
547 100042 : tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
548 100042 : }
549 :
550 : /**
551 : * Processes a file.
552 : */
553 974966 : static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, struct stat* st, uint64_t physical)
554 : {
555 974966 : struct snapraid_state* state = scan->state;
556 974966 : struct snapraid_disk* disk = scan->disk;
557 : struct snapraid_file* file;
558 : tommy_node* i;
559 : int is_original_file_size_different_than_zero;
560 : int is_file_already_present;
561 : int is_file_reported;
562 :
563 : /*
564 : * If the disk has persistent inodes and UUID, try a search on the past inodes,
565 : * to detect moved files.
566 : *
567 : * For persistent inodes we mean inodes that keep their values when the filesystem
568 : * is unmounted and remounted. This don't always happen.
569 : *
570 : * Cases found are:
571 : * - Linux FUSE with exFAT driver from https://code.google.com/p/exfat/.
572 : * Inodes are reassigned at every mount restarting from 1 and incrementing.
573 : * As worse, the exFAT support in FUSE doesn't use subsecond precision in timestamps
574 : * making inode collision more easy (exFAT by design supports 10ms precision).
575 : * - Linux VFAT kernel (3.2) driver. Inodes are fully reassigned at every mount.
576 : *
577 : * In such cases, to avoid possible random collisions, it's better to disable the moved
578 : * file recognition.
579 : *
580 : * For persistent UUID we mean that it has the same UUID as before.
581 : * Otherwise, if the UUID is changed, likely it's a new recreated filesystem,
582 : * and then the inode have no meaning.
583 : *
584 : * Note that to disable the search by past inode, we do this implicitely
585 : * removing all the past inode before searching for files.
586 : * This ensures that no file is found with a past inode, but at the same time,
587 : * it allows to find new files with the same inode, to identify them as hardlinks.
588 : */
589 974966 : int has_past_inodes = !disk->has_volatile_inodes && !disk->has_different_uuid && !disk->has_unsupported_uuid;
590 :
591 : /* always search with the new inode, in the all new inodes found until now, */
592 : /* with the eventual presence of also the past inodes */
593 974966 : uint64_t inode = st->st_ino;
594 :
595 974966 : file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
596 :
597 : /* identify moved files with past inodes and hardlinks with the new inodes */
598 974966 : if (file) {
599 : /* check if the file is not changed */
600 811432 : if (file->size == st->st_size
601 803087 : && file->mtime_sec == st->st_mtime
602 801536 : && (file->mtime_nsec == STAT_NSEC(st)
603 : /* always accept the stored value if it's STAT_NSEC_INVALID */
604 : /* it happens when upgrading from an old version of SnapRAID */
605 : /* not yet supporting the nanosecond field */
606 0 : || file->mtime_nsec == STAT_NSEC_INVALID
607 : )
608 : ) {
609 : /* check if multiple files have the same inode */
610 801536 : if (file_flag_has(file, FILE_IS_PRESENT)) {
611 : #if HAVE_STRUCT_STAT_ST_NLINK
612 326 : if (st->st_nlink <= 1) {
613 : /* LCOV_EXCL_START */
614 : log_fatal("Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub);
615 : exit(EXIT_FAILURE);
616 : /* LCOV_EXCL_STOP */
617 : }
618 : #endif
619 : /* it's a hardlink */
620 326 : scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
621 326 : return;
622 : }
623 :
624 : /* mark as present */
625 801210 : file_flag_set(file, FILE_IS_PRESENT);
626 :
627 : /* update the nanoseconds mtime only if different */
628 : /* to avoid unneeded updates */
629 801210 : if (file->mtime_nsec == STAT_NSEC_INVALID
630 0 : && STAT_NSEC(st) != file->mtime_nsec
631 : ) {
632 0 : file->mtime_nsec = STAT_NSEC(st);
633 :
634 : /* we have to save the new mtime */
635 0 : state->need_write = 1;
636 : }
637 :
638 801210 : if (strcmp(file->sub, sub) != 0) {
639 : /* if the path is different, it means a moved file with the same inode */
640 155 : ++scan->count_move;
641 :
642 155 : log_tag("scan:move:%s:%s:%s\n", disk->name, esc(file->sub), esc(sub));
643 155 : if (is_diff) {
644 0 : printf("move %s%s -> %s%s\n", disk->dir, file->sub, disk->dir, sub);
645 : }
646 :
647 : /* remove from the name set */
648 155 : tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
649 :
650 : /* save the new name */
651 155 : file_rename(file, sub);
652 :
653 : /* reinsert in the name set */
654 155 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
655 :
656 : /* we have to save the new name */
657 155 : state->need_write = 1;
658 : } else {
659 : /* otherwise it's equal */
660 801055 : ++scan->count_equal;
661 :
662 801055 : if (state->opt.gui) {
663 22555 : log_tag("scan:equal:%s:%s\n", disk->name, esc(file->sub));
664 : }
665 : }
666 :
667 : /* mark the file as kept */
668 801210 : scan_file_keep(scan, file);
669 :
670 : /* nothing more to do */
671 801210 : return;
672 : }
673 :
674 : /* here the file matches the inode, but not the other info */
675 : /* if could be a modified file with the same name, */
676 : /* or a restored/copied file that get assigned a previously used inode, */
677 : /* or a filesystem with not persistent inodes */
678 :
679 : /* for sure it cannot be already present */
680 9896 : if (file_flag_has(file, FILE_IS_PRESENT)) {
681 : /* LCOV_EXCL_START */
682 : log_fatal("Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' matching and already present but different\n", file->inode, disk->dir, sub, disk->dir, file->sub);
683 : exit(EXIT_FAILURE);
684 : /* LCOV_EXCL_STOP */
685 : }
686 :
687 : /* assume a previously used inode, it's the worst case */
688 : /* and we handle it removing the duplicate stored inode. */
689 : /* If the file is found by name (not necessarely in this function call), */
690 : /* it will have the inode restored, otherwise, it will get removed */
691 :
692 : /* remove from the inode set */
693 9896 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
694 :
695 : /* clear the inode */
696 : /* this is not really needed for correct functionality */
697 : /* because we are going to set FILE_IS_WITHOUT_INODE */
698 : /* but it's easier for debugging to have invalid inodes set to 0 */
699 9896 : file->inode = 0;
700 :
701 : /* mark as missing inode */
702 9896 : file_flag_set(file, FILE_IS_WITHOUT_INODE);
703 :
704 : /* go further to find it by name */
705 : }
706 :
707 : /* initialize for later overwrite */
708 173430 : is_file_reported = 0;
709 173430 : is_original_file_size_different_than_zero = 0;
710 :
711 : /* then try finding it by name */
712 173430 : file = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub, file_path_hash(sub));
713 :
714 : /* keep track if the file already exists */
715 173430 : is_file_already_present = file != 0;
716 :
717 173430 : if (is_file_already_present) {
718 : /* if the file is without an inode */
719 98966 : if (file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
720 : /* set it now */
721 9855 : file->inode = st->st_ino;
722 :
723 : /* insert in the set */
724 9855 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
725 :
726 : /* unmark as missing inode */
727 9855 : file_flag_clear(file, FILE_IS_WITHOUT_INODE);
728 : } else {
729 : /* here the inode has to be different, otherwise we would have found it before */
730 89111 : if (file->inode == st->st_ino) {
731 : /* LCOV_EXCL_START */
732 : log_fatal("Internal inconsistency in inode '%" PRIu64 "' for files '%s%s' as unexpected matching\n", file->inode, disk->dir, sub);
733 : exit(EXIT_FAILURE);
734 : /* LCOV_EXCL_STOP */
735 : }
736 : }
737 :
738 : /* for sure it cannot be already present */
739 98966 : if (file_flag_has(file, FILE_IS_PRESENT)) {
740 : /* LCOV_EXCL_START */
741 : log_fatal("Internal inconsistency in path for file '%s%s' matching and already present\n", disk->dir, sub);
742 : exit(EXIT_FAILURE);
743 : /* LCOV_EXCL_STOP */
744 : }
745 :
746 : /* check if the file is not changed */
747 98966 : if (file->size == st->st_size
748 98372 : && file->mtime_sec == st->st_mtime
749 96053 : && (file->mtime_nsec == STAT_NSEC(st)
750 : /* always accept the stored value if it's STAT_NSEC_INVALID */
751 : /* it happens when upgrading from an old version of SnapRAID */
752 : /* not yet supporting the nanosecond field */
753 0 : || file->mtime_nsec == STAT_NSEC_INVALID
754 : )
755 : ) {
756 : /* mark as present */
757 96053 : file_flag_set(file, FILE_IS_PRESENT);
758 :
759 : /* update the nano seconds mtime only if different */
760 : /* to avoid unneeded updates */
761 96053 : if (file->mtime_nsec == STAT_NSEC_INVALID
762 0 : && STAT_NSEC(st) != STAT_NSEC_INVALID
763 : ) {
764 0 : file->mtime_nsec = STAT_NSEC(st);
765 :
766 : /* we have to save the new mtime */
767 0 : state->need_write = 1;
768 : }
769 :
770 : /* if when processing the disk we used the past inodes values */
771 96053 : if (has_past_inodes) {
772 : /* if persistent inodes are supported, we are sure that the inode number */
773 : /* is now different, because otherwise the file would have been found */
774 : /* when searching by inode. */
775 : /* if the inode is different, it means a rewritten file with the same path */
776 : /* like when restoring a backup that restores also the timestamp */
777 96053 : ++scan->count_restore;
778 :
779 96053 : log_tag("scan:restore:%s:%s\n", disk->name, esc(sub));
780 96053 : if (is_diff) {
781 0 : printf("restore %s%s\n", disk->dir, sub);
782 : }
783 :
784 : /* remove from the inode set */
785 96053 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
786 :
787 : /* save the new inode */
788 96053 : file->inode = st->st_ino;
789 :
790 : /* reinsert in the inode set */
791 96053 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
792 :
793 : /* we have to save the new inode */
794 96053 : state->need_write = 1;
795 : } else {
796 : /* otherwise it's the case of not persistent inode, where doesn't */
797 : /* matter if the inode is different or equal, because they have no */
798 : /* meaning, and then we don't even save them */
799 0 : ++scan->count_equal;
800 :
801 0 : if (state->opt.gui) {
802 0 : log_tag("scan:equal:%s:%s\n", disk->name, esc(file->sub));
803 : }
804 : }
805 :
806 : /* mark the file as kept */
807 96053 : scan_file_keep(scan, file);
808 :
809 : /* nothing more to do */
810 96053 : return;
811 : }
812 :
813 : /* here if the file is changed but with the correct name */
814 :
815 : /* keep track if the original file was not of zero size */
816 2913 : is_original_file_size_different_than_zero = file->size != 0;
817 :
818 : /* remove it, and continue to insert it again */
819 2913 : scan_file_remove(scan, file);
820 :
821 : /* and continue to insert it again */
822 : }
823 :
824 : /* refresh the info, to ensure that they are synced, */
825 : /* note that we refresh only the info of the new or modified files */
826 : /* because this is slow operation */
827 77377 : scan_file_refresh(scan, sub, st, &physical);
828 :
829 : /* do a safety check to ensure that the common ext4 case of zeroing */
830 : /* the size of a file after a crash doesn't propagate to the backup */
831 77377 : if (is_original_file_size_different_than_zero && st->st_size == 0) {
832 0 : if (!state->opt.force_zero) {
833 : /* LCOV_EXCL_START */
834 : log_fatal("The file '%s%s' has unexpected zero size!\n", disk->dir, sub);
835 : log_fatal("It's possible that after a kernel crash this file was lost,\n");
836 : log_fatal("and you can use 'snapraid --filter %s fix' to recover it.\n", sub);
837 : if (!is_diff) {
838 : log_fatal("If this an expected condition you can '%s' anyway using 'snapraid --force-zero %s'\n", state->command, state->command);
839 : exit(EXIT_FAILURE);
840 : }
841 : /* LCOV_EXCL_STOP */
842 : }
843 : }
844 :
845 : /* insert it */
846 77377 : file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, STAT_NSEC(st), st->st_ino, physical);
847 :
848 : /* mark it as present */
849 77377 : file_flag_set(file, FILE_IS_PRESENT);
850 :
851 : /* if copy detection is enabled */
852 : /* note that the copy detection is tried also for updated files */
853 : /* this makes sense because it may happen to have two different copies */
854 : /* of the same file, and we move the right one over the wrong one */
855 : /* in such case we have a "copy" over an "update" */
856 77377 : if (!state->opt.force_nocopy) {
857 77377 : tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
858 :
859 : /* search for a file with the same name and stamp in all the disks */
860 506296 : for (i = state->disklist; i != 0; i = i->next) {
861 435064 : struct snapraid_disk* other_disk = i->data;
862 : struct snapraid_file* other_file;
863 :
864 : /* if the nanosecond part of the time stamp is valid, search */
865 : /* for name and stamp, otherwise for path and stamp */
866 435064 : if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
867 434956 : other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
868 : else
869 108 : other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
870 :
871 : /* if found, and it's a fully hashed file */
872 435064 : if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) {
873 : /* assume that the file is a copy, and reuse the hash */
874 6145 : file_copy(other_file, file);
875 :
876 : /* revert old counter and use the copy one */
877 6145 : ++scan->count_copy;
878 :
879 6145 : log_tag("scan:copy:%s:%s:%s:%s\n", other_disk->name, esc(other_file->sub), disk->name, esc(file->sub));
880 6145 : if (is_diff) {
881 31 : printf("copy %s%s -> %s%s\n", other_disk->dir, other_file->sub, disk->dir, file->sub);
882 : }
883 :
884 : /* mark it as reported */
885 6145 : is_file_reported = 1;
886 :
887 : /* no need to continue the search */
888 6145 : break;
889 : }
890 : }
891 : }
892 :
893 : /* if not yet reported, do it now */
894 77377 : if (!is_file_reported) {
895 71232 : if (is_file_already_present) {
896 2913 : ++scan->count_change;
897 :
898 2913 : log_tag("scan:update:%s:%s\n", disk->name, esc(sub));
899 2913 : if (is_diff) {
900 117 : printf("update %s%s\n", disk->dir, sub);
901 : }
902 : } else {
903 68319 : ++scan->count_insert;
904 :
905 68319 : log_tag("scan:add:%s:%s\n", disk->name, esc(sub));
906 68319 : if (is_diff) {
907 17043 : printf("add %s%s\n", disk->dir, sub);
908 : }
909 : }
910 : }
911 :
912 : /* insert the file in the file hashtables, to allow to find duplicate hardlinks */
913 77377 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
914 77377 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
915 77377 : tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
916 :
917 : /* insert the file in the delayed block allocation */
918 77377 : scan_file_delayed_insert(scan, file);
919 : }
920 :
921 : /**
922 : * Removes the specified dir from the data set.
923 : */
924 2 : static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
925 : {
926 2 : struct snapraid_state* state = scan->state;
927 2 : struct snapraid_disk* disk = scan->disk;
928 :
929 : /* state changed */
930 2 : state->need_write = 1;
931 :
932 : /* remove the file from the dir containers */
933 2 : tommy_hashdyn_remove_existing(&disk->dirset, &dir->nodeset);
934 2 : tommy_list_remove_existing(&disk->dirlist, &dir->nodelist);
935 :
936 : /* deallocate */
937 2 : dir_free(dir);
938 2 : }
939 :
940 : /**
941 : * Inserts the specified dir in the data set.
942 : */
943 10 : static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
944 : {
945 10 : struct snapraid_state* state = scan->state;
946 10 : struct snapraid_disk* disk = scan->disk;
947 :
948 : /* state changed */
949 10 : state->need_write = 1;
950 :
951 : /* insert the dir in the dir containers */
952 10 : tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
953 10 : tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
954 10 : }
955 :
956 : /**
957 : * Processes a dir.
958 : */
959 219 : static void scan_emptydir(struct snapraid_scan* scan, const char* sub)
960 : {
961 219 : struct snapraid_disk* disk = scan->disk;
962 : struct snapraid_dir* dir;
963 :
964 : /* check if the dir already exists */
965 219 : dir = tommy_hashdyn_search(&disk->dirset, dir_name_compare, sub, dir_name_hash(sub));
966 219 : if (dir) {
967 : /* check if multiple files have the same name */
968 209 : if (dir_flag_has(dir, FILE_IS_PRESENT)) {
969 : /* LCOV_EXCL_START */
970 : log_fatal("Internal inconsistency for dir '%s%s'\n", disk->dir, sub);
971 : exit(EXIT_FAILURE);
972 : /* LCOV_EXCL_STOP */
973 : }
974 :
975 : /* mark as present */
976 209 : dir_flag_set(dir, FILE_IS_PRESENT);
977 :
978 : /* nothing more to do */
979 428 : return;
980 : } else {
981 : /* and continue to insert it */
982 : }
983 :
984 : /* insert it */
985 10 : dir = dir_alloc(sub);
986 :
987 : /* mark it as present */
988 10 : dir_flag_set(dir, FILE_IS_PRESENT);
989 :
990 : /* insert it in the delayed insert list */
991 10 : tommy_list_insert_tail(&scan->dir_insert_list, &dir->nodelist, dir);
992 : }
993 :
994 : struct dirent_sorted {
995 : /* node for data structures */
996 : tommy_node node;
997 :
998 : #if HAVE_STRUCT_DIRENT_D_INO
999 : uint64_t d_ino; /**< Inode number. */
1000 : #endif
1001 : #if HAVE_STRUCT_DIRENT_D_TYPE
1002 : uint32_t d_type; /**< File type. */
1003 : #endif
1004 : #if HAVE_STRUCT_DIRENT_D_STAT
1005 : struct stat d_stat; /**< Stat result. */
1006 : #endif
1007 : char d_name[]; /**< Variable length name. It must be the last field. */
1008 : };
1009 :
1010 : #if HAVE_STRUCT_DIRENT_D_INO
1011 103300 : static int dd_ino_compare(const void* void_a, const void* void_b)
1012 : {
1013 103300 : const struct dirent_sorted* a = void_a;
1014 103300 : const struct dirent_sorted* b = void_b;
1015 :
1016 103300 : if (a->d_ino < b->d_ino)
1017 49394 : return -1;
1018 53906 : if (a->d_ino > b->d_ino)
1019 53906 : return 1;
1020 :
1021 0 : return 0;
1022 : }
1023 : #endif
1024 :
1025 10206597 : static int dd_name_compare(const void* void_a, const void* void_b)
1026 : {
1027 10206597 : const struct dirent_sorted* a = void_a;
1028 10206597 : const struct dirent_sorted* b = void_b;
1029 :
1030 10206597 : return strcmp(a->d_name, b->d_name);
1031 : }
1032 :
1033 : /**
1034 : * Returns the stat info of a dir entry.
1035 : */
1036 : #if HAVE_STRUCT_DIRENT_D_STAT
1037 : #define DSTAT(file, dd, buf) dstat(dd)
1038 : struct stat* dstat(struct dirent_sorted* dd)
1039 : {
1040 : return &dd->d_stat;
1041 : }
1042 : #else
1043 : #define DSTAT(file, dd, buf) dstat(file, buf)
1044 976055 : struct stat* dstat(const char* file, struct stat* st)
1045 : {
1046 976055 : if (lstat(file, st) != 0) {
1047 : /* LCOV_EXCL_START */
1048 : log_fatal("Error in stat file/directory '%s'. %s.\n", file, strerror(errno));
1049 : exit(EXIT_FAILURE);
1050 : /* LCOV_EXCL_STOP */
1051 : }
1052 976055 : return st;
1053 : }
1054 : #endif
1055 :
1056 : /**
1057 : * Processes a directory.
1058 : * Return != 0 if at least one file or link is processed.
1059 : */
1060 1502 : static int scan_dir(struct snapraid_scan* scan, int is_diff, const char* dir, const char* sub)
1061 : {
1062 1502 : struct snapraid_state* state = scan->state;
1063 1502 : struct snapraid_disk* disk = scan->disk;
1064 1502 : int processed = 0;
1065 : DIR* d;
1066 : tommy_list list;
1067 : tommy_node* node;
1068 :
1069 1502 : tommy_list_init(&list);
1070 :
1071 1502 : d = opendir(dir);
1072 1502 : if (!d) {
1073 : /* LCOV_EXCL_START */
1074 : log_fatal("Error opening directory '%s'. %s.\n", dir, strerror(errno));
1075 : log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub);
1076 : exit(EXIT_FAILURE);
1077 : /* LCOV_EXCL_STOP */
1078 : }
1079 :
1080 : /* read the full directory */
1081 : while (1) {
1082 : char path_next[PATH_MAX];
1083 : char sub_next[PATH_MAX];
1084 : struct dirent_sorted* entry;
1085 : const char* name;
1086 : struct dirent* dd;
1087 : size_t name_len;
1088 :
1089 : /* clear errno to detect erroneous conditions */
1090 1012101 : errno = 0;
1091 1012101 : dd = readdir(d);
1092 1012101 : if (dd == 0 && errno != 0) {
1093 : /* LCOV_EXCL_START */
1094 : log_fatal("Error reading directory '%s'. %s.\n", dir, strerror(errno));
1095 : log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub);
1096 : exit(EXIT_FAILURE);
1097 : /* LCOV_EXCL_STOP */
1098 : }
1099 1012101 : if (dd == 0) {
1100 1502 : break; /* finished */
1101 : }
1102 :
1103 : /* skip "." and ".." files */
1104 1010599 : name = dd->d_name;
1105 1010599 : if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
1106 6008 : continue;
1107 :
1108 1007595 : pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
1109 1007595 : pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
1110 :
1111 : /* check for not supported file names */
1112 1007595 : if (name[0] == 0) {
1113 : /* LCOV_EXCL_START */
1114 : log_fatal("Unsupported name '%s' in file '%s'.\n", name, path_next);
1115 : exit(EXIT_FAILURE);
1116 : /* LCOV_EXCL_STOP */
1117 : }
1118 :
1119 : /* exclude hidden files even before calling lstat() */
1120 1007595 : if (filter_hidden(state->filter_hidden, dd) != 0) {
1121 0 : msg_verbose("Excluding hidden '%s'\n", path_next);
1122 0 : continue;
1123 : }
1124 :
1125 : /* exclude content files even before calling lstat() */
1126 1007595 : if (filter_content(&state->contentlist, path_next) != 0) {
1127 0 : msg_verbose("Excluding content '%s'\n", path_next);
1128 0 : continue;
1129 : }
1130 :
1131 1007595 : name_len = strlen(dd->d_name);
1132 1007595 : entry = malloc_nofail(sizeof(struct dirent_sorted) + name_len + 1);
1133 :
1134 : /* copy the dir entry */
1135 : #if HAVE_STRUCT_DIRENT_D_INO
1136 1007595 : entry->d_ino = dd->d_ino;
1137 : #endif
1138 : #if HAVE_STRUCT_DIRENT_D_TYPE
1139 1007595 : entry->d_type = dd->d_type;
1140 : #endif
1141 : #if HAVE_STRUCT_DIRENT_D_STAT
1142 : /* convert dirent to lstat result */
1143 : dirent_lstat(dd, &entry->d_stat);
1144 :
1145 : /* note that at this point the st_mode may be 0 */
1146 : #endif
1147 1007595 : memcpy(entry->d_name, dd->d_name, name_len + 1);
1148 :
1149 : /* insert in the list */
1150 1007595 : tommy_list_insert_tail(&list, &entry->node, entry);
1151 1010599 : }
1152 :
1153 1502 : if (closedir(d) != 0) {
1154 : /* LCOV_EXCL_START */
1155 : log_fatal("Error closing directory '%s'. %s.\n", dir, strerror(errno));
1156 : exit(EXIT_FAILURE);
1157 : /* LCOV_EXCL_STOP */
1158 : }
1159 :
1160 1502 : if (state->opt.force_order == SORT_ALPHA) {
1161 : /* if requested sort alphabetically */
1162 : /* this is mainly done for testing to ensure to always */
1163 : /* process in the same way in different platforms */
1164 1466 : tommy_list_sort(&list, dd_name_compare);
1165 : }
1166 : #if HAVE_STRUCT_DIRENT_D_INO
1167 36 : else if (!disk->has_volatile_inodes) {
1168 : /* if inodes are persistent */
1169 : /* sort the list of dir entries by inodes */
1170 36 : tommy_list_sort(&list, dd_ino_compare);
1171 : }
1172 : /* otherwise just keep the insertion order */
1173 : #endif
1174 :
1175 : /* process the sorted dir entries */
1176 1502 : node = list;
1177 1010599 : while (node != 0) {
1178 : char path_next[PATH_MAX];
1179 : char sub_next[PATH_MAX];
1180 : char out[PATH_MAX];
1181 1007595 : struct snapraid_filter* reason = 0;
1182 1007595 : struct dirent_sorted* dd = node->data;
1183 1007595 : const char* name = dd->d_name;
1184 : struct stat* st;
1185 : int type;
1186 : #if !HAVE_STRUCT_DIRENT_D_STAT
1187 : struct stat st_buf;
1188 : #endif
1189 :
1190 1007595 : pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
1191 1007595 : pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
1192 :
1193 : /* start with an unknown type */
1194 1007595 : type = -1;
1195 1007595 : st = 0;
1196 :
1197 : /* if dirent has the type, use it */
1198 : #if HAVE_STRUCT_DIRENT_D_TYPE
1199 1007595 : switch (dd->d_type) {
1200 0 : case DT_UNKNOWN : break;
1201 974966 : case DT_REG : type = 0; break;
1202 31540 : case DT_LNK : type = 1; break;
1203 1089 : case DT_DIR : type = 2; break;
1204 0 : default : type = 3; break;
1205 : }
1206 : #endif
1207 :
1208 : /* if type is still unknown */
1209 1007595 : if (type < 0) {
1210 : /* get the type from stat */
1211 0 : st = DSTAT(path_next, dd, &st_buf);
1212 :
1213 : #if HAVE_STRUCT_DIRENT_D_STAT
1214 : /* if the st_mode field is missing, takes care to fill it using normal lstat() */
1215 : /* at now this can happen only in Windows (with HAVE_STRUCT_DIRENT_D_STAT defined), */
1216 : /* because we use a directory reading method that doesn't read info about ReparsePoint. */
1217 : /* Note that here we cannot call here lstat_sync(), because we don't know what kind */
1218 : /* of file is it, and lstat_sync() doesn't always work */
1219 : if (st->st_mode == 0) {
1220 : if (lstat(path_next, st) != 0) {
1221 : /* LCOV_EXCL_START */
1222 : log_fatal("Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
1223 : exit(EXIT_FAILURE);
1224 : /* LCOV_EXCL_STOP */
1225 : }
1226 : }
1227 : #endif
1228 :
1229 0 : if (S_ISREG(st->st_mode))
1230 0 : type = 0;
1231 0 : else if (S_ISLNK(st->st_mode))
1232 0 : type = 1;
1233 0 : else if (S_ISDIR(st->st_mode))
1234 0 : type = 2;
1235 : else
1236 0 : type = 3;
1237 : }
1238 :
1239 1007595 : if (type == 0) { /* REG */
1240 974966 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1241 :
1242 : /* late stat, if not yet called */
1243 974966 : if (!st)
1244 974966 : st = DSTAT(path_next, dd, &st_buf);
1245 :
1246 : #if HAVE_LSTAT_SYNC
1247 : /* if the st_ino field is missing, takes care to fill it using the extended lstat() */
1248 : /* this can happen only in Windows */
1249 : if (st->st_ino == 0) {
1250 : if (lstat_sync(path_next, st, 0) != 0) {
1251 : /* LCOV_EXCL_START */
1252 : log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
1253 : exit(EXIT_FAILURE);
1254 : /* LCOV_EXCL_STOP */
1255 : }
1256 : }
1257 : #endif
1258 :
1259 974966 : scan_file(scan, is_diff, sub_next, st, FILEPHY_UNREAD_OFFSET);
1260 974966 : processed = 1;
1261 : } else {
1262 0 : msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
1263 : }
1264 32629 : } else if (type == 1) { /* LNK */
1265 31540 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1266 : char subnew[PATH_MAX];
1267 : int ret;
1268 :
1269 31540 : ret = readlink(path_next, subnew, sizeof(subnew));
1270 31540 : if (ret >= PATH_MAX) {
1271 : /* LCOV_EXCL_START */
1272 : log_fatal("Error in readlink file '%s'. Symlink too long.\n", path_next);
1273 : exit(EXIT_FAILURE);
1274 : /* LCOV_EXCL_STOP */
1275 : }
1276 31540 : if (ret < 0) {
1277 : /* LCOV_EXCL_START */
1278 : log_fatal("Error in readlink file '%s'. %s.\n", path_next, strerror(errno));
1279 : exit(EXIT_FAILURE);
1280 : /* LCOV_EXCL_STOP */
1281 : }
1282 :
1283 : /* readlink doesn't put the final 0 */
1284 31540 : subnew[ret] = 0;
1285 :
1286 : /* process as a symbolic link */
1287 31540 : scan_link(scan, is_diff, sub_next, subnew, FILE_IS_SYMLINK);
1288 31540 : processed = 1;
1289 : } else {
1290 0 : msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
1291 : }
1292 1089 : } else if (type == 2) { /* DIR */
1293 1089 : if (filter_dir(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1294 : #ifndef _WIN32
1295 : /* late stat, if not yet called */
1296 1089 : if (!st)
1297 1089 : st = DSTAT(path_next, dd, &st_buf);
1298 :
1299 : /* in Unix don't follow mount points in different devices */
1300 : /* in Windows we are already skipping them reporting them as special files */
1301 1089 : if ((uint64_t)st->st_dev != disk->device) {
1302 0 : log_fatal("WARNING! Ignoring mount point '%s' because it appears to be in a different device\n", path_next);
1303 : } else
1304 : #endif
1305 : {
1306 : char sub_dir[PATH_MAX];
1307 :
1308 : /* recurse */
1309 1089 : pathslash(path_next, sizeof(path_next));
1310 1089 : pathcpy(sub_dir, sizeof(sub_dir), sub_next);
1311 1089 : pathslash(sub_dir, sizeof(sub_dir));
1312 1089 : if (scan_dir(scan, is_diff, path_next, sub_dir) == 0) {
1313 : /* scan the directory as empty dir */
1314 219 : scan_emptydir(scan, sub_next);
1315 : }
1316 : /* or we processed something internally, or we have added the empty dir */
1317 1089 : processed = 1;
1318 : }
1319 : } else {
1320 0 : msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
1321 : }
1322 : } else {
1323 0 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1324 : /* late stat, if not yet called */
1325 0 : if (!st)
1326 0 : st = DSTAT(path_next, dd, &st_buf);
1327 :
1328 0 : log_fatal("WARNING! Ignoring special '%s' file '%s'\n", stat_desc(st), path_next);
1329 : } else {
1330 0 : msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
1331 : }
1332 : }
1333 :
1334 : /* next entry */
1335 1007595 : node = node->next;
1336 :
1337 : /* free the present one */
1338 1007595 : free(dd);
1339 : }
1340 :
1341 1502 : return processed;
1342 : }
1343 :
1344 69 : static int state_diffscan(struct snapraid_state* state, int is_diff)
1345 : {
1346 : tommy_node* i;
1347 : tommy_node* j;
1348 : tommy_list scanlist;
1349 : int done;
1350 : fptr* msg;
1351 : struct snapraid_scan total;
1352 : int no_difference;
1353 :
1354 69 : tommy_list_init(&scanlist);
1355 :
1356 69 : if (is_diff)
1357 6 : msg_progress("Comparing...\n");
1358 :
1359 : /* first scan all the directory and find new and deleted files */
1360 482 : for (i = state->disklist; i != 0; i = i->next) {
1361 413 : struct snapraid_disk* disk = i->data;
1362 : struct snapraid_scan* scan;
1363 : tommy_node* node;
1364 : int ret;
1365 : int has_persistent_inode;
1366 :
1367 413 : scan = malloc_nofail(sizeof(struct snapraid_scan));
1368 413 : scan->state = state;
1369 413 : scan->disk = disk;
1370 413 : scan->count_equal = 0;
1371 413 : scan->count_move = 0;
1372 413 : scan->count_copy = 0;
1373 413 : scan->count_restore = 0;
1374 413 : scan->count_change = 0;
1375 413 : scan->count_remove = 0;
1376 413 : scan->count_insert = 0;
1377 413 : tommy_list_init(&scan->file_insert_list);
1378 413 : tommy_list_init(&scan->link_insert_list);
1379 413 : tommy_list_init(&scan->dir_insert_list);
1380 :
1381 413 : tommy_list_insert_tail(&scanlist, &scan->node, scan);
1382 :
1383 413 : if (!is_diff)
1384 377 : msg_progress("Scanning disk %s...\n", disk->name);
1385 :
1386 : /* check if the disk supports persistent inodes */
1387 413 : ret = fsinfo(disk->dir, &has_persistent_inode, 0, 0);
1388 413 : if (ret < 0) {
1389 : /* LCOV_EXCL_START */
1390 : log_fatal("Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno));
1391 : exit(EXIT_FAILURE);
1392 : /* LCOV_EXCL_STOP */
1393 : }
1394 413 : if (!has_persistent_inode) {
1395 0 : disk->has_volatile_inodes = 1;
1396 : }
1397 :
1398 : /* if inodes or UUID are not persistent/changed/unsupported */
1399 413 : if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
1400 : /* removes all the inodes from the inode collection */
1401 : /* if they are not persistent, all of them could be changed now */
1402 : /* and we don't want to find false matching ones */
1403 : /* see scan_file() for more details */
1404 37 : node = disk->filelist;
1405 74 : while (node) {
1406 0 : struct snapraid_file* file = node->data;
1407 :
1408 0 : node = node->next;
1409 :
1410 : /* remove from the inode set */
1411 0 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
1412 :
1413 : /* clear the inode */
1414 0 : file->inode = 0;
1415 :
1416 : /* mark as missing inode */
1417 0 : file_flag_set(file, FILE_IS_WITHOUT_INODE);
1418 : }
1419 : }
1420 :
1421 413 : scan_dir(scan, is_diff, disk->dir, "");
1422 : }
1423 :
1424 : /* we split the search in two phases because to detect files */
1425 : /* moved from one disk to another we have to start deletion */
1426 : /* only when all disks have all the new files found */
1427 :
1428 : /* now process all the new and deleted files */
1429 482 : for (i = scanlist; i != 0; i = i->next) {
1430 413 : struct snapraid_scan* scan = i->data;
1431 413 : struct snapraid_disk* disk = scan->disk;
1432 : tommy_node* node;
1433 : unsigned phy_count;
1434 : unsigned phy_dup;
1435 : uint64_t phy_last;
1436 : struct snapraid_file* phy_file_last;
1437 :
1438 : /* check for removed files */
1439 413 : node = disk->filelist;
1440 905053 : while (node) {
1441 904227 : struct snapraid_file* file = node->data;
1442 :
1443 : /* next node */
1444 904227 : node = node->next;
1445 :
1446 : /* remove if not present */
1447 904227 : if (!file_flag_has(file, FILE_IS_PRESENT)) {
1448 29629 : ++scan->count_remove;
1449 :
1450 29629 : log_tag("scan:remove:%s:%s\n", disk->name, esc(file->sub));
1451 29629 : if (is_diff) {
1452 97 : printf("remove %s%s\n", disk->dir, file->sub);
1453 : }
1454 :
1455 29629 : scan_file_remove(scan, file);
1456 : }
1457 : }
1458 :
1459 : /* check for removed links */
1460 413 : node = disk->linklist;
1461 31191 : while (node) {
1462 30365 : struct snapraid_link* link = node->data;
1463 :
1464 : /* next node */
1465 30365 : node = node->next;
1466 :
1467 : /* remove if not present */
1468 30365 : if (!link_flag_has(link, FILE_IS_PRESENT)) {
1469 956 : ++scan->count_remove;
1470 :
1471 956 : log_tag("scan:remove:%s:%s\n", disk->name, esc(link->sub));
1472 956 : if (is_diff) {
1473 9 : printf("remove %s%s\n", disk->dir, link->sub);
1474 : }
1475 :
1476 956 : scan_link_remove(scan, link);
1477 : }
1478 : }
1479 :
1480 : /* check for removed dirs */
1481 413 : node = disk->dirlist;
1482 1037 : while (node) {
1483 211 : struct snapraid_dir* dir = node->data;
1484 :
1485 : /* next node */
1486 211 : node = node->next;
1487 :
1488 : /* remove if not present */
1489 211 : if (!dir_flag_has(dir, FILE_IS_PRESENT)) {
1490 2 : scan_emptydir_remove(scan, dir);
1491 : }
1492 : }
1493 :
1494 : /* sort the files before inserting them */
1495 : /* we use a stable sort to ensure that if the reported physical offset/inode */
1496 : /* are always 0, we keep at least the directory order */
1497 413 : switch (state->opt.force_order) {
1498 : case SORT_PHYSICAL :
1499 12 : tommy_list_sort(&scan->file_insert_list, file_physical_compare);
1500 12 : break;
1501 : case SORT_INODE :
1502 0 : tommy_list_sort(&scan->file_insert_list, file_inode_compare);
1503 0 : break;
1504 : case SORT_ALPHA :
1505 401 : tommy_list_sort(&scan->file_insert_list, file_path_compare);
1506 401 : break;
1507 : case SORT_DIR :
1508 : /* already in order */
1509 0 : break;
1510 : }
1511 :
1512 : /* insert all the new files, we insert them only after the deletion */
1513 : /* to reuse the just freed space */
1514 : /* also check if the physical offset reported are fakes or not */
1515 413 : node = scan->file_insert_list;
1516 413 : phy_count = 0;
1517 413 : phy_dup = 0;
1518 413 : phy_last = FILEPHY_UNREAD_OFFSET;
1519 413 : phy_file_last = 0;
1520 100868 : while (node) {
1521 100042 : struct snapraid_file* file = node->data;
1522 :
1523 : /* if the file is not empty, count duplicate physical offsets */
1524 100042 : if (state->opt.force_order == SORT_PHYSICAL && file->size != 0) {
1525 11418 : if (phy_file_last != 0 && file->physical == phy_last
1526 : /* files without offset are expected to have duplicates */
1527 0 : && phy_last != FILEPHY_WITHOUT_OFFSET
1528 : ) {
1529 : /* if verbose, prints the list of duplicates real offsets */
1530 : /* other cases are for offsets not supported, so we don't need to report them file by file */
1531 0 : if (phy_last >= FILEPHY_REAL_OFFSET) {
1532 0 : log_fatal("WARNING! Files '%s%s' and '%s%s' have the same physical offset %" PRId64 ".\n", disk->dir, phy_file_last->sub, disk->dir, file->sub, phy_last);
1533 : }
1534 0 : ++phy_dup;
1535 : }
1536 11418 : phy_file_last = file;
1537 11418 : phy_last = file->physical;
1538 11418 : ++phy_count;
1539 : }
1540 :
1541 : /* next node */
1542 100042 : node = node->next;
1543 :
1544 : /* insert it */
1545 100042 : scan_file_insert(scan, file);
1546 : }
1547 :
1548 : /* mark the disk without reliable physical offset if it has duplicates */
1549 : /* here it should never happen because we already sorted out hardlinks */
1550 413 : if (state->opt.force_order == SORT_PHYSICAL && phy_dup > 0) {
1551 0 : disk->has_unreliable_physical = 1;
1552 : }
1553 :
1554 : /* insert all the new links */
1555 413 : node = scan->link_insert_list;
1556 3283 : while (node) {
1557 2457 : struct snapraid_link* link = node->data;
1558 :
1559 : /* next node */
1560 2457 : node = node->next;
1561 :
1562 : /* insert it */
1563 2457 : scan_link_insert(scan, link);
1564 : }
1565 :
1566 : /* insert all the new dirs */
1567 413 : node = scan->dir_insert_list;
1568 836 : while (node) {
1569 10 : struct snapraid_dir* dir = node->data;
1570 :
1571 : /* next node */
1572 10 : node = node->next;
1573 :
1574 : /* insert it */
1575 10 : scan_emptydir_insert(scan, dir);
1576 : }
1577 : }
1578 :
1579 : /* checks for disks where all the previously existing files where removed */
1580 69 : if (!state->opt.force_empty) {
1581 68 : int all_missing = 0;
1582 68 : int all_rewritten = 0;
1583 68 : done = 0;
1584 475 : for (i = state->disklist, j = scanlist; i != 0; i = i->next, j = j->next) {
1585 407 : struct snapraid_disk* disk = i->data;
1586 407 : struct snapraid_scan* scan = j->data;
1587 :
1588 407 : if (scan->count_equal == 0
1589 37 : && scan->count_move == 0
1590 37 : && scan->count_restore == 0
1591 37 : && (scan->count_remove != 0 || scan->count_change != 0)
1592 : ) {
1593 0 : if (!done) {
1594 0 : done = 1;
1595 0 : log_fatal("WARNING! All the files previously present in disk '%s' at dir '%s'", disk->name, disk->dir);
1596 : } else {
1597 0 : log_fatal(", disk '%s' at dir '%s'", disk->name, disk->dir);
1598 : }
1599 :
1600 : /* detect the special condition of all files missing */
1601 0 : if (scan->count_change == 0)
1602 0 : all_missing = 1;
1603 :
1604 : /* detect the special condition of all files rewritten */
1605 0 : if (scan->count_remove == 0)
1606 0 : all_rewritten = 1;
1607 : }
1608 : }
1609 68 : if (done) {
1610 0 : log_fatal("\nare now missing or rewritten!\n");
1611 0 : if (all_rewritten) {
1612 0 : log_fatal("This could happen when restoring a disk with a backup\n");
1613 0 : log_fatal("program that is not setting correctly the timestamps.\n");
1614 : }
1615 0 : if (all_missing) {
1616 0 : log_fatal("This could happen when some disks are not mounted\n");
1617 0 : log_fatal("in the expected directory.\n");
1618 : }
1619 0 : if (!is_diff) {
1620 0 : log_fatal("If you want to '%s' anyway, use 'snapraid --force-empty %s'.", state->command, state->command);
1621 0 : exit(EXIT_FAILURE);
1622 : }
1623 : }
1624 : }
1625 :
1626 : /* checks for disks without the physical offset support */
1627 69 : if (state->opt.force_order == SORT_PHYSICAL) {
1628 2 : done = 0;
1629 14 : for (i = state->disklist; i != 0; i = i->next) {
1630 12 : struct snapraid_disk* disk = i->data;
1631 :
1632 12 : if (disk->has_unreliable_physical) {
1633 0 : if (!done) {
1634 0 : done = 1;
1635 0 : log_fatal("WARNING! Physical offsets not supported for disk '%s'", disk->name);
1636 : } else {
1637 0 : log_fatal(", '%s'", disk->name);
1638 : }
1639 : }
1640 : }
1641 2 : if (done) {
1642 0 : log_fatal(". Files order won't be optimal.\n");
1643 : }
1644 : }
1645 :
1646 : /* checks for disks without persistent inodes */
1647 69 : done = 0;
1648 482 : for (i = state->disklist; i != 0; i = i->next) {
1649 413 : struct snapraid_disk* disk = i->data;
1650 :
1651 413 : if (disk->has_volatile_inodes) {
1652 0 : if (!done) {
1653 0 : done = 1;
1654 0 : log_fatal("WARNING! Inodes are not persistent for disks: '%s'", disk->name);
1655 : } else {
1656 0 : log_fatal(", '%s'", disk->name);
1657 : }
1658 : }
1659 : }
1660 69 : if (done) {
1661 0 : log_fatal(". Move operations won't be optimal.\n");
1662 : }
1663 :
1664 : /* checks for disks with changed UUID */
1665 69 : done = 0;
1666 482 : for (i = state->disklist; i != 0; i = i->next) {
1667 413 : struct snapraid_disk* disk = i->data;
1668 :
1669 : /* don't print the message if the UUID changed because before */
1670 : /* it was no set. */
1671 : /* this is the normal condition for an empty disk because it */
1672 : /* isn't stored */
1673 413 : if (disk->has_different_uuid && !disk->had_empty_uuid) {
1674 0 : if (!done) {
1675 0 : done = 1;
1676 0 : log_fatal("WARNING! UUID is changed for disks: '%s'", disk->name);
1677 : } else {
1678 0 : log_fatal(", '%s'", disk->name);
1679 : }
1680 : }
1681 : }
1682 69 : if (done) {
1683 0 : log_fatal(". Move operations won't be optimal.\n");
1684 : }
1685 :
1686 : /* checks for disks with unsupported UUID */
1687 69 : done = 0;
1688 482 : for (i = state->disklist; i != 0; i = i->next) {
1689 413 : struct snapraid_disk* disk = i->data;
1690 :
1691 413 : if (disk->has_unsupported_uuid) {
1692 0 : if (!done) {
1693 0 : done = 1;
1694 0 : log_fatal("WARNING! UUID is unsupported for disks: '%s'", disk->name);
1695 : } else {
1696 0 : log_fatal(", '%s'", disk->name);
1697 : }
1698 : }
1699 : }
1700 69 : if (done) {
1701 0 : log_fatal(". Move operations won't be optimal.\n");
1702 : }
1703 :
1704 69 : total.count_equal = 0;
1705 69 : total.count_move = 0;
1706 69 : total.count_copy = 0;
1707 69 : total.count_restore = 0;
1708 69 : total.count_change = 0;
1709 69 : total.count_remove = 0;
1710 69 : total.count_insert = 0;
1711 :
1712 482 : for (i = scanlist; i != 0; i = i->next) {
1713 413 : struct snapraid_scan* scan = i->data;
1714 413 : total.count_equal += scan->count_equal;
1715 413 : total.count_move += scan->count_move;
1716 413 : total.count_copy += scan->count_copy;
1717 413 : total.count_restore += scan->count_restore;
1718 413 : total.count_change += scan->count_change;
1719 413 : total.count_remove += scan->count_remove;
1720 413 : total.count_insert += scan->count_insert;
1721 : }
1722 :
1723 69 : if (is_diff) {
1724 6 : msg_status("\n");
1725 6 : msg = msg_status;
1726 : } else {
1727 63 : msg = msg_verbose;
1728 : }
1729 :
1730 69 : msg("%8u equal\n", total.count_equal);
1731 69 : msg("%8u moved\n", total.count_move);
1732 69 : msg("%8u copied\n", total.count_copy);
1733 69 : msg("%8u restored\n", total.count_restore);
1734 69 : msg("%8u updated\n", total.count_change);
1735 69 : msg("%8u removed\n", total.count_remove);
1736 69 : msg("%8u added\n", total.count_insert);
1737 :
1738 69 : log_tag("summary:equal:%u\n", total.count_equal);
1739 69 : log_tag("summary:moved:%u\n", total.count_move);
1740 69 : log_tag("summary:copied:%u\n", total.count_copy);
1741 69 : log_tag("summary:restored:%u\n", total.count_restore);
1742 69 : log_tag("summary:updated:%u\n", total.count_change);
1743 69 : log_tag("summary:removed:%u\n", total.count_remove);
1744 69 : log_tag("summary:added:%u\n", total.count_insert);
1745 :
1746 206 : no_difference = !total.count_move && !total.count_copy && !total.count_restore
1747 109 : && !total.count_change && !total.count_remove && !total.count_insert;
1748 :
1749 69 : if (is_diff) {
1750 6 : if (no_difference) {
1751 1 : msg_status("No differences\n");
1752 : } else {
1753 5 : msg_status("There are differences!\n");
1754 : }
1755 : }
1756 :
1757 69 : if (no_difference) {
1758 7 : log_tag("summary:exit:equal\n");
1759 : } else {
1760 62 : log_tag("summary:exit:diff\n");
1761 : }
1762 69 : log_flush();
1763 :
1764 69 : tommy_list_foreach(&scanlist, (tommy_foreach_func*)free);
1765 :
1766 69 : if (is_diff) {
1767 : /* check for file difference */
1768 6 : if (!no_difference)
1769 5 : return 1;
1770 :
1771 : /* check also for incomplete "sync" */
1772 1 : if (parity_is_invalid(state))
1773 0 : return 1;
1774 : }
1775 :
1776 64 : return 0;
1777 : }
1778 :
1779 6 : int state_diff(struct snapraid_state* state)
1780 : {
1781 6 : return state_diffscan(state, 1);
1782 : }
1783 :
1784 63 : void state_scan(struct snapraid_state* state)
1785 : {
1786 63 : (void)state_diffscan(state, 0); /* ignore return value */
1787 63 : }
1788 :
|