Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #ifndef __ELEM_H
19 : #define __ELEM_H
20 :
21 : #include "util.h"
22 : #include "support.h"
23 : #include "tommyds/tommyhash.h"
24 : #include "tommyds/tommylist.h"
25 : #include "tommyds/tommytree.h"
26 : #include "tommyds/tommyhashdyn.h"
27 : #include "tommyds/tommyarray.h"
28 : #include "tommyds/tommyarrayblkof.h"
29 :
30 : /****************************************************************************/
31 : /* snapraid */
32 :
33 : /**
34 : * Number of measures of the operation progress.
35 : */
36 : #define PROGRESS_MAX 100
37 :
38 : /**
39 : * Max UUID length.
40 : */
41 : #define UUID_MAX 128
42 :
43 : /**
44 : * Invalid position.
45 : */
46 : #define POS_NULL ((block_off_t)-1)
47 :
48 : /**
49 : * Content file specification.
50 : */
51 : struct snapraid_content {
52 : char content[PATH_MAX]; /**< Path of the content file. */
53 : uint64_t device; /**< Device identifier. */
54 : void* context; /**< Context used for multithread operations. */
55 : tommy_node node; /**< Next node in the list. */
56 : };
57 :
58 : /**
59 : * Filter for paths.
60 : */
61 : struct snapraid_filter {
62 : char pattern[PATH_MAX]; /**< Filter pattern. */
63 : int is_disk; /**< If the pattern is a disk one. */
64 : int is_path; /**< If the pattern is only for the complete path. */
65 : int is_dir; /**< If the pattern is only for dir. */
66 : int direction; /**< If it's an inclusion (=1) or an exclusion (=-1). */
67 : tommy_node node; /**< Next node in the list. */
68 : };
69 :
70 : /**
71 : * Block pointer used to represent unused blocks.
72 : */
73 : #define BLOCK_NULL 0
74 :
75 : /**
76 : * This block is an empty one.
77 : * Note that an empty block is represent with ::BLOCK_NULL.
78 : */
79 : #define BLOCK_STATE_EMPTY 0
80 :
81 : /**
82 : * The block has both the hash and the parity computed.
83 : * This is the normal state of a saved block.
84 : *
85 : * The block hash field IS set.
86 : * The parity for this disk is updated.
87 : */
88 : #define BLOCK_STATE_BLK 1
89 :
90 : /**
91 : * The block is new and not yet hashed.
92 : * This happens when a new block overwrite a just removed block, or an empty space.
93 : *
94 : * The block hash field MAY be set and it represents the hash of the OLD data.
95 : * The hash may be also INVALID or ZERO.
96 : *
97 : * If the OLD block was empty, the hash is set to the special ZERO value.
98 : * If the OLD block was lost, the hash is set to the special INVALID value.
99 : *
100 : * The parity for this disk is not updated, but it contains the old data referenced by the hash.
101 : *
102 : * If the state is read from an incomplete sync, we don't really know if the hash is referring at the
103 : * data used to compute the parity, because the sync process was interrupted at an unknown point,
104 : * and the parity may or may not be updated.
105 : *
106 : * For this reason we clear all such hashes when reading the state from an incomplete sync before
107 : * starting a new sync, because sync is affected by such hashes, as sync updates the parity, only
108 : * if the new data read for CHG blocks has a mismatching hash.
109 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
110 : * No clearing is done in other commands, as check and fix are instead able to work with unsynced
111 : * hashes, and scrub ignores CHG/DELETED blocks.
112 : */
113 : #define BLOCK_STATE_CHG 2
114 :
115 : /**
116 : * The block is new and hashed.
117 : * This happens when a new block overwrite a just removed block, or an empty space.
118 : *
119 : * Note that when the file copy heuristic is enabled, the REP blocks may be set
120 : * using this heuristic, meaning that the hash may be wrong.
121 : *
122 : * For this reason, when the ::force_nocopy flag is enabled in sync, we convert all the REP blocks
123 : * to CHG, invalidating the stored hash.
124 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
125 : * No clearing is done in other commands, as they don't stop the process like in sync
126 : * when there is a false silent error.
127 : *
128 : * The block hash field IS set, and it represents the hash of the new data.
129 : * The parity for this disk is not updated.
130 : */
131 : #define BLOCK_STATE_REP 3
132 :
133 : /**
134 : * This block is a deleted one.
135 : * This happens when a file is deleted.
136 : *
137 : * The block hash field IS set, and it represents the hash of the previous data,
138 : * but only if it's different by all 0.
139 : * The parity for this disk is not updated, but it contains the old data referenced by the hash.
140 : *
141 : * If the state is read from an incomplete sync, we don't really know if the hash is referring at the
142 : * data used to compute the parity, because the sync process was interrupted at an unknown point,
143 : * and the parity may or may not be updated.
144 : *
145 : * A now the sync process is not affected by DELETED hash, so clearing won't be really needed,
146 : * but considering that we have to do it for CHG blocks, we do it also for DELETED ones,
147 : * clearing all the past hashes.
148 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
149 : * No clearing is done in other commands, as check and fix are instead able to work with unsynced
150 : * hashes, and scrub ignores CHG/DELETED blocks.
151 : */
152 : #define BLOCK_STATE_DELETED 4
153 :
154 : /**
155 : * Block hash size.
156 : *
157 : * At max HASH_MAX.
158 : */
159 : extern int BLOCK_HASH_SIZE;
160 :
161 : /**
162 : * Block of a file.
163 : */
164 : struct snapraid_block {
165 : unsigned char state; /**< State of the block. */
166 :
167 : /**
168 : * Hash of the block.
169 : *
170 : * The effective stored size is BLOCK_HASH_SIZE.
171 : */
172 : unsigned char hash[HASH_MAX];
173 : };
174 :
175 : /**
176 : * If a file is present in the disk.
177 : * It's used only in scan to detect present and missing files.
178 : */
179 : #define FILE_IS_PRESENT 0x01
180 :
181 : /**
182 : * If it's an excluded file from the processing.
183 : * It's used in both check and fix to mark files to exclude from the processing.
184 : */
185 : #define FILE_IS_EXCLUDED 0x02
186 :
187 : /**
188 : * If a fix was attempted but it failed.
189 : * It's used only in fix to mark that some data is unrecoverable.
190 : */
191 : #define FILE_IS_DAMAGED 0x04
192 :
193 : /**
194 : * If a fix was done.
195 : * It's used only in fix to mark that some data was recovered.
196 : */
197 : #define FILE_IS_FIXED 0x08
198 :
199 : /**
200 : * If the file was originally missing, and it was created in the fix process.
201 : * It's used only in fix to mark files recovered from scratch,
202 : * meaning that they don't have any previous content.
203 : * This is important because it means that deleting them, you are not going
204 : * to lose something that cannot be recovered.
205 : * Note that excluded files won't ever get this flag.
206 : */
207 : #define FILE_IS_CREATED 0x10
208 :
209 : /**
210 : * If the file has completed its processing, meaning that it won't be opened anymore.
211 : * It's used only in fix to mark when we finish processing one file.
212 : * Note that excluded files won't ever get this flag.
213 : */
214 : #define FILE_IS_FINISHED 0x20
215 :
216 : /**
217 : * If the file hash was obtained from a file copy
218 : * identified by the same name, size and stamp.
219 : */
220 : #define FILE_IS_COPY 0x40
221 :
222 : /**
223 : * If the file was opened.
224 : * It's used in fix to detect if it's the first time a file is opened.
225 : */
226 : #define FILE_IS_OPENED 0x80
227 :
228 : /**
229 : * If the file is modified from the latest sync.
230 : * It's used in fix to store if the state of the file before being modified.
231 : */
232 : #define FILE_IS_UNSYNCED 0x100
233 :
234 : /**
235 : * If the file is without inode.
236 : * It could happen in file-system where inodes are not persistent,
237 : * or when restoring a full disk with "fix".
238 : * In such cases we have to clear any stored duplicate inode.
239 : * After the scan process completes, no file should have this flag set.
240 : */
241 : #define FILE_IS_WITHOUT_INODE 0x200
242 :
243 : /**
244 : * The file is deleted.
245 : * This happens when a file is deleted from the array,
246 : * but it's keep inside the parity until the next sync.
247 : *
248 : * During the file-system check we needs this information,
249 : * because deleted files may be present only partially.
250 : */
251 : #define FILE_IS_DELETED 0x400
252 :
253 : /**
254 : * The file is missing.
255 : * This happens in fix/check when a file is cannot be opened,
256 : * and marking it as such prevents to retry to open it again.
257 : */
258 : #define FILE_IS_MISSING 0x800
259 :
260 : #define FILE_IS_HARDLINK 0x1000 /**< If it's an hardlink. */
261 : #define FILE_IS_SYMLINK 0x2000 /**< If it's a file symlink. */
262 : #define FILE_IS_SYMDIR 0x4000 /**< If it's a dir symlink for Windows. Not yet supported. */
263 : #define FILE_IS_JUNCTION 0x8000 /**< If it's a junction for Windows. Not yet supported. */
264 : #define FILE_IS_LINK_MASK 0xF000 /**< Mask for link type. */
265 :
266 : /**
267 : * File.
268 : */
269 : struct snapraid_file {
270 : int64_t mtime_sec; /**< Modification time. */
271 : uint64_t inode; /**< Inode. */
272 : uint64_t physical; /**< Physical offset of the file. */
273 : data_off_t size; /**< Size of the file. */
274 : struct snapraid_block* blockvec; /**< All the blocks of the file. */
275 : int mtime_nsec; /**< Modification time nanoseconds. In the range 0 <= x < 1,000,000,000, or STAT_NSEC_INVALID if not present. */
276 : block_off_t blockmax; /**< Number of blocks. */
277 : unsigned flag; /**< FILE_IS_* flags. */
278 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
279 :
280 : /* nodes for data structures */
281 : tommy_node nodelist;
282 : tommy_hashdyn_node nodeset;
283 : tommy_hashdyn_node pathset;
284 : tommy_hashdyn_node stampset;
285 : };
286 :
287 : /**
288 : * Symbolic Link.
289 : */
290 : struct snapraid_link {
291 : unsigned flag; /**< FILE_IS_* flags. */
292 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
293 : char* linkto; /**< Link to. */
294 :
295 : /* nodes for data structures */
296 : tommy_node nodelist;
297 : tommy_hashdyn_node nodeset;
298 : };
299 :
300 : /**
301 : * Dir.
302 : */
303 : struct snapraid_dir {
304 : unsigned flag; /**< FILE_IS_* flags. */
305 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
306 :
307 : /* nodes for data structures */
308 : tommy_node nodelist;
309 : tommy_hashdyn_node nodeset;
310 : };
311 :
312 : /**
313 : * Chunk.
314 : *
315 : * A extent represents a fragment of a file mapped into the parity.
316 : */
317 : struct snapraid_extent {
318 : struct snapraid_file* file; /**< File containing this extent. */
319 : block_off_t parity_pos; /**< Parity position. */
320 : block_off_t file_pos; /**< Position in the file. */
321 : block_off_t count; /**< Number of sequential blocks in the file and parity. */
322 : tommy_tree_node parity_node; /**< Tree sorted by <parity_pos>. */
323 : tommy_tree_node file_node; /**< Tree sorter by <file,file_pos>. */
324 : };
325 :
326 : /**
327 : * Disk.
328 : */
329 : struct snapraid_disk {
330 : char name[PATH_MAX]; /**< Name of the disk. */
331 : char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */
332 : char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
333 : char uuid[UUID_MAX]; /**< UUID of the disk. */
334 :
335 : uint64_t device; /**< Device identifier. */
336 : block_off_t total_blocks; /**< Number of total blocks. */
337 : block_off_t free_blocks; /**< Number of free blocks at the last sync. */
338 :
339 : uint64_t tick; /**< Usage time. */
340 : uint64_t progress_tick[PROGRESS_MAX]; /**< Last ticks of progress. */
341 : unsigned cached; /**< Number of IO blocks cached. */
342 :
343 : /**
344 : * First free searching block.
345 : * Note that it doesn't necessarily point at the first free block,
346 : * but it just tell you that no free block is present before this position.
347 : */
348 : block_off_t first_free_block;
349 :
350 : int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */
351 : int has_unreliable_physical; /**< If the physical offset of files has duplicates. */
352 : int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */
353 : int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */
354 : int had_empty_uuid; /**< If the disk had an empty UUID, meaning that it's a new disk. */
355 : int mapping_idx; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
356 : int skip_access; /**< If the disk is inaccessible and it should be skipped. */
357 :
358 : #if HAVE_PTHREAD
359 : /**
360 : * Mutex for protecting the filesystem structure.
361 : *
362 : * Specifically, this protects ::fs_parity, ::fs_file, and ::fs_last,
363 : * meaning that it protects only extents.
364 : *
365 : * Files, links and dirs are not protected as they are not expected to
366 : * change during multithread processing.
367 : */
368 : pthread_mutex_t fs_mutex;
369 : #endif
370 :
371 : /**
372 : * Mapping of extents in the parity.
373 : * Sorted by <parity_pos> and by <file,file_pos>
374 : */
375 : tommy_tree fs_parity;
376 : tommy_tree fs_file;
377 :
378 : /**
379 : * Last extent we accessed.
380 : * It's used to optimize access of sequential blocks.
381 : */
382 : struct snapraid_extent* fs_last;
383 :
384 : /**
385 : * List of all the snapraid_file for the disk.
386 : */
387 : tommy_list filelist;
388 :
389 : /**
390 : * List of all the deleted file for the disk.
391 : *
392 : * These files are kept allocated, because the blocks are still referenced in
393 : * the ::blockarr.
394 : */
395 : tommy_list deletedlist;
396 :
397 : tommy_hashdyn inodeset; /**< Hashtable by inode of all the files. */
398 : tommy_hashdyn pathset; /**< Hashtable by path of all the files. */
399 : tommy_hashdyn stampset; /**< Hashtable by stamp (size and time) of all the files. */
400 : tommy_list linklist; /**< List of all the links. */
401 : tommy_hashdyn linkset; /**< Hashtable by name of all the links. */
402 : tommy_list dirlist; /**< List of all the empty dirs. */
403 : tommy_hashdyn dirset; /**< Hashtable by name of all the empty dirs. */
404 :
405 : /* nodes for data structures */
406 : tommy_node node;
407 : };
408 :
409 : /**
410 : * Disk mapping.
411 : */
412 : struct snapraid_map {
413 : char name[PATH_MAX]; /**< Name of the disk. */
414 : char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
415 : block_off_t total_blocks; /**< Number of total blocks. */
416 : block_off_t free_blocks; /**< Number of free blocks at last 'sync'. */
417 : unsigned position; /**< Position of the disk in the parity. */
418 :
419 : /* nodes for data structures */
420 : tommy_node node;
421 : };
422 :
423 : /**
424 : * Max number of parity split.
425 : */
426 : #define SPLIT_MAX 8
427 :
428 : /**
429 : * Invalid parity size.
430 : *
431 : * This value is used to identify new parities,
432 : * like when you alter the configuration adding
433 : * a new parity level, creating it with 'fix'.
434 : * Given that 'fix' doesn't write the content file,
435 : * the new size will be written only at the next
436 : * 'sync'.
437 : */
438 : #define PARITY_SIZE_INVALID -1LL
439 :
440 : /**
441 : * Parity split.
442 : */
443 : struct snapraid_split {
444 : char path[PATH_MAX]; /**< Path of the parity file. */
445 : char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
446 :
447 : /**
448 : * Size of the parity split.
449 : * Only the latest not zero size is allowed to grow.
450 : * If the value is unset, it's PARITY_SIZE_INVALID.
451 : */
452 : data_off_t size;
453 :
454 : uint64_t device; /**< Device identifier of the parity. */
455 : };
456 :
457 : /**
458 : * Parity.
459 : */
460 : struct snapraid_parity {
461 : struct snapraid_split split_map[SPLIT_MAX]; /**< Parity splits. */
462 : unsigned split_mac; /**< Number of parity splits. */
463 : char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
464 : block_off_t total_blocks; /**< Number of total blocks. */
465 : block_off_t free_blocks; /**< Number of free blocks at the last sync. */
466 : int is_excluded; /**< If the parity is excluded by disk filter. */
467 : int skip_access; /**< If at least one of the parity disk is inaccessible and it should be skipped. */
468 : uint64_t tick; /**< Usage time. */
469 : uint64_t progress_tick[PROGRESS_MAX]; /**< Last cpu ticks of progress. */
470 : unsigned cached; /**< Number of IO blocks cached. */
471 : };
472 :
473 : /**
474 : * Info.
475 : */
476 : typedef uint32_t snapraid_info;
477 :
478 : /**
479 : * Allocate a content.
480 : */
481 : struct snapraid_content* content_alloc(const char* path, uint64_t dev);
482 :
483 : /**
484 : * Deallocate a content.
485 : */
486 : void content_free(struct snapraid_content* content);
487 :
488 : /**
489 : * Allocate a filter pattern for files and directories.
490 : */
491 : struct snapraid_filter* filter_alloc_file(int is_include, const char* pattern);
492 :
493 : /**
494 : * Allocate a filter pattern for disks.
495 : */
496 : struct snapraid_filter* filter_alloc_disk(int is_include, const char* pattern);
497 :
498 : /**
499 : * Deallocate an exclusion.
500 : */
501 : void filter_free(struct snapraid_filter* filter);
502 :
503 : /**
504 : * Filter type description.
505 : */
506 : const char* filter_type(struct snapraid_filter* filter, char* out, size_t out_size);
507 :
508 : /**
509 : * Filter hidden files.
510 : * Return !=0 if it matches and it should be excluded.
511 : */
512 3173372 : static inline int filter_hidden(int enable, struct dirent* dd)
513 : {
514 3173372 : if (enable && dirent_hidden(dd)) {
515 0 : return 1; /* filter out */
516 : }
517 :
518 3173372 : return 0;
519 : }
520 :
521 : /**
522 : * Filter a path using a list of filters.
523 : * For each element of the path all the filters are applied, until the first one that matches.
524 : * Return !=0 if it should be excluded.
525 : */
526 : int filter_path(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
527 :
528 : /**
529 : * Filter a file/link/dir if missing.
530 : * This call imply a disk check for the file presence.
531 : * Return !=0 if the file is present and it should be excluded.
532 : */
533 : int filter_existence(int filter_missing, const char* dir, const char* sub);
534 :
535 : /**
536 : * Filter a file if bad.
537 : * Return !=0 if the file is correct and it should be excluded.
538 : */
539 : int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file);
540 :
541 : /**
542 : * Filter a dir using a list of filters.
543 : * For each element of the path all the filters are applied, until the first one that matches.
544 : * Thesesdir are always by included by default, to allow to apply rules at the contained files.
545 : * Return !=0 if should be excluded.
546 : */
547 : int filter_subdir(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
548 :
549 : /**
550 : * Filter a dir using a list of filters.
551 : * For each element of the path all the filters are applied, until the first one that matches.
552 : * Return !=0 if should be excluded.
553 : */
554 : int filter_emptydir(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
555 :
556 : /**
557 : * Filter a path if it's a content file.
558 : * Return !=0 if should be excluded.
559 : */
560 : int filter_content(tommy_list* contentlist, const char* path);
561 :
562 : /**
563 : * Check if the specified hash is invalid.
564 : *
565 : * An invalid hash is represented with all bytes at 0x00.
566 : *
567 : * If working with reduced hash lengths, this function always return 0.
568 : */
569 758645 : static inline int hash_is_invalid(const unsigned char* hash)
570 : {
571 : int i;
572 :
573 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
574 758645 : if (BLOCK_HASH_SIZE != HASH_MAX)
575 0 : return 0;
576 :
577 1178483 : for (i = 0; i < BLOCK_HASH_SIZE; ++i)
578 1152419 : if (hash[i] != 0x00)
579 732581 : return 0;
580 :
581 26064 : return 1;
582 : }
583 :
584 : /**
585 : * Check if the specified hash represent the zero block.
586 : *
587 : * A zero hash is represented with all bytes at 0xFF.
588 : *
589 : * If working with reduced hash lengths, this function always return 0.
590 : */
591 760108 : static inline int hash_is_zero(const unsigned char* hash)
592 : {
593 : int i;
594 :
595 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
596 760108 : if (BLOCK_HASH_SIZE != HASH_MAX)
597 0 : return 0;
598 :
599 994848 : for (i = 0; i < BLOCK_HASH_SIZE; ++i)
600 980378 : if (hash[i] != 0xFF)
601 745638 : return 0;
602 :
603 14470 : return 1;
604 : }
605 :
606 : /**
607 : * Check if the specified hash is unequivocally representing the data.
608 : *
609 : * If working with reduced hash lengths, this function always return 0.
610 : */
611 44270 : static inline int hash_is_unique(const unsigned char* hash)
612 : {
613 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
614 44270 : if (BLOCK_HASH_SIZE != HASH_MAX)
615 0 : return 0;
616 :
617 44270 : return !hash_is_zero(hash) && !hash_is_invalid(hash);
618 : }
619 :
620 : /**
621 : * Set the hash to the special INVALID value.
622 : */
623 9234067 : static inline void hash_invalid_set(unsigned char* hash)
624 : {
625 9234067 : memset(hash, 0x00, BLOCK_HASH_SIZE);
626 9234067 : }
627 :
628 : /**
629 : * Set the hash to the special ZERO value.
630 : */
631 115543 : static inline void hash_zero_set(unsigned char* hash)
632 : {
633 115543 : memset(hash, 0xFF, BLOCK_HASH_SIZE);
634 115543 : }
635 :
636 : /**
637 : * Allocated space for block.
638 : */
639 58065128 : static inline size_t block_sizeof(void)
640 : {
641 58065128 : return 1 + BLOCK_HASH_SIZE;
642 : }
643 :
644 : /**
645 : * Get the state of the block.
646 : *
647 : * For this function, it's allowed to pass a NULL block
648 : * pointer than results in the BLOCK_STATE_EMPTY state.
649 : */
650 52304023 : static inline unsigned block_state_get(const struct snapraid_block* block)
651 : {
652 52304023 : if (block == BLOCK_NULL)
653 1722497 : return BLOCK_STATE_EMPTY;
654 :
655 50581526 : return block->state;
656 : }
657 :
658 : /**
659 : * Set the state of the block.
660 : */
661 19117096 : static inline void block_state_set(struct snapraid_block* block, unsigned state)
662 : {
663 19117096 : block->state = state;
664 19117096 : }
665 :
666 : /**
667 : * Check if the specified block has an updated hash.
668 : *
669 : * Note that EMPTY / CHG / DELETED return 0.
670 : */
671 2217099 : static inline int block_has_updated_hash(const struct snapraid_block* block)
672 : {
673 2217099 : unsigned state = block_state_get(block);
674 :
675 2217099 : return state == BLOCK_STATE_BLK || state == BLOCK_STATE_REP;
676 : }
677 :
678 : /**
679 : * Check if the specified block has a past hash,
680 : * i.e. the hash of the data that it's now overwritten or lost.
681 : *
682 : * Note that EMPTY / BLK / REP return 0.
683 : */
684 2542004 : static inline int block_has_past_hash(const struct snapraid_block* block)
685 : {
686 2542004 : unsigned state = block_state_get(block);
687 :
688 2542004 : return state == BLOCK_STATE_CHG || state == BLOCK_STATE_DELETED;
689 : }
690 :
691 : /**
692 : * Check if the specified block is part of a file.
693 : *
694 : * Note that EMPTY / DELETED return 0.
695 : */
696 14591493 : static inline int block_has_file(const struct snapraid_block* block)
697 : {
698 14591493 : unsigned state = block_state_get(block);
699 :
700 14591444 : return state == BLOCK_STATE_BLK
701 14591444 : || state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
702 : }
703 :
704 : /**
705 : * Check if the block has an invalid parity than needs to be updated.
706 : *
707 : * Note that EMPTY / BLK return 0.
708 : */
709 12320585 : static inline int block_has_invalid_parity(const struct snapraid_block* block)
710 : {
711 12320585 : unsigned state = block_state_get(block);
712 :
713 12320585 : return state == BLOCK_STATE_DELETED
714 12320585 : || state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
715 : }
716 :
717 : /**
718 : * Check if the block is part of a file with valid parity.
719 : *
720 : * Note that anything different than BLK return 0.
721 : */
722 220962 : static inline int block_has_file_and_valid_parity(const struct snapraid_block* block)
723 : {
724 220962 : unsigned state = block_state_get(block);
725 :
726 220962 : return state == BLOCK_STATE_BLK;
727 : }
728 :
729 34089332 : static inline int file_flag_has(const struct snapraid_file* file, unsigned mask)
730 : {
731 34089332 : return (file->flag & mask) == mask;
732 : }
733 :
734 5295048 : static inline void file_flag_set(struct snapraid_file* file, unsigned mask)
735 : {
736 5295048 : file->flag |= mask;
737 5295048 : }
738 :
739 11286 : static inline void file_flag_clear(struct snapraid_file* file, unsigned mask)
740 : {
741 11286 : file->flag &= ~mask;
742 11286 : }
743 :
744 : /**
745 : * Allocate a file.
746 : */
747 : struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical);
748 :
749 : /**
750 : * Duplicate a file.
751 : */
752 : struct snapraid_file* file_dup(struct snapraid_file* copy);
753 :
754 : /**
755 : * Deallocate a file.
756 : */
757 : void file_free(struct snapraid_file* file);
758 :
759 : /**
760 : * Rename a file.
761 : */
762 : void file_rename(struct snapraid_file* file, const char* sub);
763 :
764 : /**
765 : * Copy a file.
766 : */
767 : void file_copy(struct snapraid_file* src_file, struct snapraid_file* dest_file);
768 :
769 : /**
770 : * Return the block at the specified position.
771 : *
772 : * Note that the block size if a runtime value.
773 : */
774 54351316 : static inline struct snapraid_block* file_block(struct snapraid_file* file, size_t pos)
775 : {
776 54351316 : unsigned char* ptr = (unsigned char*)file->blockvec;
777 :
778 54351316 : return (struct snapraid_block*)(ptr + pos * block_sizeof());
779 : }
780 :
781 : /**
782 : * Return the name of the file, without the dir.
783 : */
784 : const char* file_name(const struct snapraid_file* file);
785 :
786 : /**
787 : * Check if the block is the last in the file.
788 : */
789 : int file_block_is_last(struct snapraid_file* file, block_off_t file_pos);
790 :
791 : /**
792 : * Get the size in bytes of the block.
793 : * If it's the last block of a file it could be less than block_size.
794 : */
795 : unsigned file_block_size(struct snapraid_file* file, block_off_t file_pos, unsigned block_size);
796 :
797 : /**
798 : * Compare a file with an inode.
799 : */
800 : int file_inode_compare_to_arg(const void* void_arg, const void* void_data);
801 :
802 : /**
803 : * Compare files by inode.
804 : */
805 : int file_inode_compare(const void* void_a, const void* void_b);
806 :
807 : /**
808 : * Compare files by path.
809 : */
810 : int file_path_compare(const void* void_a, const void* void_b);
811 :
812 : /**
813 : * Compare files by physical address.
814 : */
815 : int file_physical_compare(const void* void_a, const void* void_b);
816 :
817 : /**
818 : * Compute the hash of a file inode.
819 : */
820 5042449 : static inline tommy_uint32_t file_inode_hash(uint64_t inode)
821 : {
822 5042449 : return (tommy_uint32_t)tommy_inthash_u64(inode);
823 : }
824 :
825 : /**
826 : * Compare a file with a path.
827 : */
828 : int file_path_compare_to_arg(const void* void_arg, const void* void_data);
829 :
830 : /**
831 : * Compare a file with another file for name, stamp, and both.
832 : */
833 : int file_name_compare(const void* void_a, const void* void_b);
834 : int file_stamp_compare(const void* void_a, const void* void_b);
835 : int file_namestamp_compare(const void* void_a, const void* void_b);
836 : int file_pathstamp_compare(const void* void_a, const void* void_b);
837 :
838 : /**
839 : * Compute the hash of a file path.
840 : */
841 3886182 : static inline tommy_uint32_t file_path_hash(const char* sub)
842 : {
843 3886182 : return tommy_hash_u32(0, sub, strlen(sub));
844 : }
845 :
846 : /**
847 : * Compute the hash of a file stamp.
848 : */
849 6275421 : static inline tommy_uint32_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec)
850 : {
851 6275421 : return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32(mtime_sec ^ tommy_inthash_u32(mtime_nsec)));
852 : }
853 :
854 : /**
855 : * Allocate a extent.
856 : */
857 : struct snapraid_extent* extent_alloc(block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos, block_off_t count);
858 :
859 : /**
860 : * Deallocate a extent.
861 : */
862 : void extent_free(struct snapraid_extent* extent);
863 :
864 : /**
865 : * Compare extent by parity position.
866 : */
867 : int extent_parity_compare(const void* void_a, const void* void_b);
868 :
869 : /**
870 : * Compare extent by file and file position.
871 : */
872 : int extent_file_compare(const void* void_a, const void* void_b);
873 :
874 209328 : static inline int link_flag_has(const struct snapraid_link* slink, unsigned mask)
875 : {
876 209328 : return (slink->flag & mask) == mask;
877 : }
878 :
879 39461 : static inline void link_flag_set(struct snapraid_link* slink, unsigned mask)
880 : {
881 39461 : slink->flag |= mask;
882 39461 : }
883 :
884 : static inline void link_flag_clear(struct snapraid_link* slink, unsigned mask)
885 : {
886 : slink->flag &= ~mask;
887 : }
888 :
889 6 : static inline void link_flag_let(struct snapraid_link* slink, unsigned flag, unsigned mask)
890 : {
891 6 : slink->flag &= ~mask;
892 6 : slink->flag |= flag & mask;
893 6 : }
894 :
895 91794 : static inline unsigned link_flag_get(struct snapraid_link* slink, unsigned mask)
896 : {
897 91794 : return slink->flag & mask;
898 : }
899 :
900 : /**
901 : * Allocate a link.
902 : */
903 : struct snapraid_link* link_alloc(const char* name, const char* slink, unsigned link_flag);
904 :
905 : /**
906 : * Deallocate a link.
907 : */
908 : void link_free(struct snapraid_link* slink);
909 :
910 : /**
911 : * Compare a link with a name.
912 : */
913 : int link_name_compare_to_arg(const void* void_arg, const void* void_data);
914 :
915 : /**
916 : * Compare links by path.
917 : */
918 : int link_alpha_compare(const void* void_a, const void* void_b);
919 :
920 : /**
921 : * Compute the hash of a link name.
922 : */
923 155706 : static inline tommy_uint32_t link_name_hash(const char* name)
924 : {
925 155706 : return tommy_hash_u32(0, name, strlen(name));
926 : }
927 :
928 940 : static inline int dir_flag_has(const struct snapraid_dir* dir, unsigned mask)
929 : {
930 940 : return (dir->flag & mask) == mask;
931 : }
932 :
933 278 : static inline void dir_flag_set(struct snapraid_dir* dir, unsigned mask)
934 : {
935 278 : dir->flag |= mask;
936 278 : }
937 :
938 : static inline void dir_flag_clear(struct snapraid_dir* dir, unsigned mask)
939 : {
940 : dir->flag &= ~mask;
941 : }
942 :
943 : /**
944 : * Allocate a dir.
945 : */
946 : struct snapraid_dir* dir_alloc(const char* name);
947 :
948 : /**
949 : * Deallocate a dir.
950 : */
951 : void dir_free(struct snapraid_dir* dir);
952 :
953 : /**
954 : * Compare a dir with a name.
955 : */
956 : int dir_name_compare(const void* void_arg, const void* void_data);
957 :
958 : /**
959 : * Compute the hash of a dir name.
960 : */
961 1059 : static inline tommy_uint32_t dir_name_hash(const char* name)
962 : {
963 1059 : return tommy_hash_u32(0, name, strlen(name));
964 : }
965 :
966 : /**
967 : * Allocate a disk.
968 : */
969 : struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid, int skip);
970 :
971 : /**
972 : * Deallocate a disk.
973 : */
974 : void disk_free(struct snapraid_disk* disk);
975 :
976 : /**
977 : * Get the size of the disk in blocks.
978 : */
979 : block_off_t fs_size(struct snapraid_disk* disk);
980 :
981 : /**
982 : * Check if a disk is totally empty and can be discarded from the content file.
983 : * A disk is empty if it doesn't contain any file, symlink, hardlink or dir
984 : * and without any DELETED block.
985 : * The blockmax is used to limit the search of DELETED block up to blockmax.
986 : */
987 : int fs_is_empty(struct snapraid_disk* disk, block_off_t blockmax);
988 :
989 : /**
990 : * Check the file-system for errors.
991 : * Return 0 if it's OK.
992 : */
993 : int fs_check(struct snapraid_disk* disk);
994 :
995 : /**
996 : * Allocate a parity position for the specified file position.
997 : *
998 : * After this call you can use the par2file/par2block operations
999 : * to query the relation.
1000 : *
1001 : * \note This function is NOT thread-safe as it uses the the disk cache. +
1002 : */
1003 : void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos);
1004 :
1005 : /**
1006 : * Deallocate the parity position.
1007 : *
1008 : * After this call the par2file/par2block operations
1009 : * won't find anymore the parity association.
1010 : *
1011 : * \note This function is NOT thread-safe as it uses the the disk cache.
1012 : */
1013 : void fs_deallocate(struct snapraid_disk* disk, block_off_t pos);
1014 :
1015 : /**
1016 : * Get the block from the file position.
1017 : */
1018 : struct snapraid_block* fs_file2block_get(struct snapraid_file* file, block_off_t file_pos);
1019 :
1020 : /**
1021 : * Get the file position from the parity position.
1022 : * Return 0 if no file is using it.
1023 : */
1024 : struct snapraid_file* fs_par2file_find(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos);
1025 :
1026 : /**
1027 : * Get the file position from the parity position.
1028 : */
1029 11723835 : static inline struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos)
1030 : {
1031 : struct snapraid_file* ret;
1032 :
1033 11723835 : ret = fs_par2file_find(disk, parity_pos, file_pos);
1034 11724079 : if (ret == 0) {
1035 : /* LCOV_EXCL_START */
1036 : log_fatal("Internal inconsistency when deresolving parity to file at position '%u' in disk '%s'\n", parity_pos, disk->name);
1037 : os_abort();
1038 : /* LCOV_EXCL_STOP */
1039 : }
1040 :
1041 11724079 : return ret;
1042 : }
1043 :
1044 : /**
1045 : * Get the parity position from the file position.
1046 : * Return POS_NULL if no parity is allocated.
1047 : */
1048 : block_off_t fs_file2par_find(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos);
1049 :
1050 : /**
1051 : * Get the parity position from the file position.
1052 : */
1053 4755492 : static inline block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
1054 : {
1055 : block_off_t ret;
1056 :
1057 4755492 : ret = fs_file2par_find(disk, file, file_pos);
1058 4755492 : if (ret == POS_NULL) {
1059 : /* LCOV_EXCL_START */
1060 : log_fatal("Internal inconsistency when resolving file '%s' at position '%u/%u' in disk '%s'\n", file->sub, file_pos, file->blockmax, disk->name);
1061 : os_abort();
1062 : /* LCOV_EXCL_STOP */
1063 : }
1064 :
1065 4755492 : return ret;
1066 : }
1067 :
1068 : /**
1069 : * Get the block from the parity position.
1070 : * Return BLOCK_NULL==0 if the block is over the end of the disk or not used.
1071 : */
1072 : struct snapraid_block* fs_par2block_find(struct snapraid_disk* disk, block_off_t parity_pos);
1073 :
1074 : /**
1075 : * Get the block from the parity position.
1076 : */
1077 15867 : static inline struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos)
1078 : {
1079 : struct snapraid_block* ret;
1080 :
1081 15867 : ret = fs_par2block_find(disk, parity_pos);
1082 15867 : if (ret == BLOCK_NULL) {
1083 : /* LCOV_EXCL_START */
1084 : log_fatal("Internal inconsistency when deresolving parity to block at position '%u' in disk '%s'\n", parity_pos, disk->name);
1085 : os_abort();
1086 : /* LCOV_EXCL_STOP */
1087 : }
1088 :
1089 15867 : return ret;
1090 : }
1091 :
1092 : /**
1093 : * Allocate a disk mapping.
1094 : * Uses uuid="" if not available.
1095 : */
1096 : struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid);
1097 :
1098 : /**
1099 : * Deallocate a disk mapping.
1100 : */
1101 : void map_free(struct snapraid_map* map);
1102 :
1103 : /**
1104 : * Mask used to store additional information in the info bits.
1105 : *
1106 : * These bits reduce the granularity of the time in the memory representation.
1107 : */
1108 : #define INFO_MASK 0x7
1109 :
1110 : /**
1111 : * Make an info.
1112 : */
1113 194583 : static inline snapraid_info info_make(time_t last_access, int error, int rehash, int justsynced)
1114 : {
1115 : /* clear the lowest bits as reserved for other information */
1116 194583 : snapraid_info info = last_access & ~INFO_MASK;
1117 :
1118 194583 : if (error != 0)
1119 5278 : info |= 0x1;
1120 194583 : if (rehash != 0)
1121 14426 : info |= 0x2;
1122 194583 : if (justsynced != 0)
1123 143045 : info |= 0x4;
1124 194583 : return info;
1125 : }
1126 :
1127 : /**
1128 : * Extract the time information.
1129 : * This is the last time when the block was know to be correct.
1130 : * The "scrubbed" info tells if the time is referreing at the latest sync or scrub.
1131 : */
1132 883203 : static inline time_t info_get_time(snapraid_info info)
1133 : {
1134 883203 : return info & ~INFO_MASK;
1135 : }
1136 :
1137 : /**
1138 : * Extract the error information.
1139 : * Report if the block address had some problem.
1140 : */
1141 352076 : static inline int info_get_bad(snapraid_info info)
1142 : {
1143 352076 : return (info & 0x1) != 0;
1144 : }
1145 :
1146 : /**
1147 : * Extract the rehash information.
1148 : * Report if the block address is using the old hash and needs to be rehashed.
1149 : */
1150 2046851 : static inline int info_get_rehash(snapraid_info info)
1151 : {
1152 2046851 : return (info & 0x2) != 0;
1153 : }
1154 :
1155 : /**
1156 : * Extract the scrubbed information.
1157 : * Report if the block address was never scrubbed.
1158 : */
1159 68634 : static inline int info_get_justsynced(snapraid_info info)
1160 : {
1161 68634 : return (info & 0x4) != 0;
1162 : }
1163 :
1164 : /**
1165 : * Mark the block address as with error.
1166 : */
1167 1772 : static inline snapraid_info info_set_bad(snapraid_info info)
1168 : {
1169 1772 : return info | 0x1;
1170 : }
1171 :
1172 : /**
1173 : * Mark the block address as with rehash.
1174 : */
1175 9215 : static inline snapraid_info info_set_rehash(snapraid_info info)
1176 : {
1177 9215 : return info | 0x2;
1178 : }
1179 :
1180 : /**
1181 : * Set the info at the specified position.
1182 : * The position is allocated if not yet done.
1183 : */
1184 1698121 : static inline void info_set(tommy_arrayblkof* array, block_off_t pos, snapraid_info info)
1185 : {
1186 1698121 : tommy_arrayblkof_grow(array, pos + 1);
1187 :
1188 1698121 : memcpy(tommy_arrayblkof_ref(array, pos), &info, sizeof(snapraid_info));
1189 1698121 : }
1190 :
1191 : /**
1192 : * Get the info at the specified position.
1193 : * For not allocated position, 0 is returned.
1194 : */
1195 5098669 : static inline snapraid_info info_get(tommy_arrayblkof* array, block_off_t pos)
1196 : {
1197 : snapraid_info info;
1198 :
1199 5098669 : if (pos >= tommy_arrayblkof_size(array))
1200 262643 : return 0;
1201 :
1202 4836026 : memcpy(&info, tommy_arrayblkof_ref(array, pos), sizeof(snapraid_info));
1203 :
1204 4836026 : return info;
1205 : }
1206 :
1207 : /**
1208 : * Compare times
1209 : */
1210 : int time_compare(const void* void_a, const void* void_b);
1211 :
1212 : /****************************************************************************/
1213 : /* format */
1214 :
1215 : #define FMT_FILE 0 /**< Print only the file. */
1216 : #define FMT_DISK 1 /**< Print the disk name and the file. */
1217 : #define FMT_PATH 2 /**< Print the full path. */
1218 :
1219 : extern int FMT_MODE;
1220 :
1221 : /**
1222 : * Format a file path for poll reference
1223 : */
1224 : const char* fmt_poll(const struct snapraid_disk* disk, const char* str, char* buffer);
1225 :
1226 : /**
1227 : * Format a path name for terminal reference
1228 : */
1229 : const char* fmt_term(const struct snapraid_disk* disk, const char* str, char* buffer);
1230 :
1231 : #endif
1232 :
|