Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #ifndef __ELEM_H
19 : #define __ELEM_H
20 :
21 : #include "util.h"
22 : #include "support.h"
23 : #include "tommyds/tommyhash.h"
24 : #include "tommyds/tommylist.h"
25 : #include "tommyds/tommytree.h"
26 : #include "tommyds/tommyhashdyn.h"
27 : #include "tommyds/tommyarray.h"
28 : #include "tommyds/tommyarrayblkof.h"
29 :
30 : /****************************************************************************/
31 : /* snapraid */
32 :
33 : /**
34 : * Number of measures of the operation progress.
35 : */
36 : #define PROGRESS_MAX 100
37 :
38 : /**
39 : * Max UUID length.
40 : */
41 : #define UUID_MAX 128
42 :
43 : /**
44 : * Max filesystem type/label length
45 : */
46 :
47 : #define FSINFO_MAX 64
48 :
49 : /**
50 : * Invalid position.
51 : */
52 : #define POS_NULL ((block_off_t)-1)
53 :
54 : /**
55 : * Content file specification.
56 : */
57 : struct snapraid_content {
58 : char content[PATH_MAX]; /**< Path of the content file. */
59 : uint64_t device; /**< Device identifier. */
60 : void* context; /**< Context used for multithread operations. */
61 : tommy_node node; /**< Next node in the list. */
62 : };
63 :
64 : /**
65 : * Filter for paths.
66 : */
67 : struct snapraid_filter {
68 : char pattern[PATH_MAX]; /**< Filter pattern. */
69 : char root[PATH_MAX]; /**< If empty, it's a global pattern. If not empty, it's a local pattern that applies only to that dir. */
70 : int is_disk; /**< If the pattern is a disk one, otherwise it's a file pattern */
71 : int is_abs; /**< If the pattern is an absolute path (start with /), otherwise it's a relative one */
72 : int is_dir; /**< If the pattern is only for dir, otherwise it's only for file. */
73 : int direction; /**< If it's an inclusion (=1) or an exclusion (=-1). */
74 : tommy_node node; /**< Next node in the list. */
75 : };
76 :
77 : /**
78 : * Block pointer used to represent unused blocks.
79 : */
80 : #define BLOCK_NULL 0
81 :
82 : /**
83 : * This block is an empty one.
84 : * Note that an empty block is represent with ::BLOCK_NULL.
85 : */
86 : #define BLOCK_STATE_EMPTY 0
87 :
88 : /**
89 : * The block has both the hash and the parity computed.
90 : * This is the normal state of a saved block.
91 : *
92 : * The block hash field IS set.
93 : * The parity for this disk is updated.
94 : */
95 : #define BLOCK_STATE_BLK 1
96 :
97 : /**
98 : * The block is new and not yet hashed.
99 : * This happens when a new block overwrites a just removed block, or an empty
100 : * space.
101 : *
102 : * The block hash field MAY be set and it represents the hash of the OLD data.
103 : * The hash may be also INVALID or ZERO.
104 : *
105 : * If the OLD block was empty, the hash is set to the special ZERO value.
106 : * If the OLD block was lost, the hash is set to the special INVALID value.
107 : *
108 : * The parity for this disk is not updated, but it contains the old data referenced by the hash.
109 : *
110 : * If the state is read from an incomplete sync, we don't really know if the hash is referring to the
111 : * data used to compute the parity, because the sync process was interrupted at an unknown point,
112 : * and the parity may or may not be updated.
113 : *
114 : * For this reason we clear all such hashes when reading the state from an incomplete sync before
115 : * starting a new sync, because sync is affected by such hashes, as sync updates the parity, only
116 : * if the new data read for CHG blocks has a mismatching hash.
117 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
118 : * No clearing is done in other commands, as check and fix are instead able to work with unsynced
119 : * hashes, and scrub ignores CHG/DELETED blocks.
120 : */
121 : #define BLOCK_STATE_CHG 2
122 :
123 : /**
124 : * The block is new and hashed.
125 : * This happens when a new block overwrites a just removed block, or an empty space.
126 : *
127 : * Note that when the file copy heuristic is enabled, the REP blocks may be set
128 : * using this heuristic, meaning that the hash may be wrong.
129 : *
130 : * For this reason, when the ::force_nocopy flag is enabled in sync, we convert all the REP blocks
131 : * to CHG, invalidating the stored hash.
132 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
133 : * No clearing is done in other commands, as they don't stop the process like in sync
134 : * when there is a false silent error.
135 : *
136 : * The block hash field IS set, and it represents the hash of the new data.
137 : * The parity for this disk is not updated.
138 : */
139 : #define BLOCK_STATE_REP 3
140 :
141 : /**
142 : * This block is a deleted one.
143 : * This happens when a file is deleted.
144 : *
145 : * The block hash field IS set, and it represents the hash of the previous data,
146 : * but only if it's different by all 0.
147 : * The parity for this disk is not updated, but it contains the old data referenced by the hash.
148 : *
149 : * If the state is read from an incomplete sync, we don't really know if the hash is referring to the
150 : * data used to compute the parity, because the sync process was interrupted at an unknown point,
151 : * and the parity may or may not be updated.
152 : *
153 : * As now the sync process is not affected by DELETED hash, so clearing won't be really needed,
154 : * but considering that we have to do it for CHG blocks, we do it also for DELETED ones,
155 : * clearing all the past hashes.
156 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
157 : * No clearing is done in other commands, as check and fix are instead able to work with unsynced
158 : * hashes, and scrub ignores CHG/DELETED blocks.
159 : */
160 : #define BLOCK_STATE_DELETED 4
161 :
162 : /**
163 : * Block hash size.
164 : *
165 : * At max HASH_MAX.
166 : */
167 : extern int BLOCK_HASH_SIZE;
168 :
169 : /**
170 : * Block of a file.
171 : */
172 : struct snapraid_block {
173 : unsigned char state; /**< State of the block. */
174 :
175 : /**
176 : * Hash of the block.
177 : *
178 : * The effective stored size is BLOCK_HASH_SIZE.
179 : */
180 : unsigned char hash[HASH_MAX];
181 : };
182 :
183 : /**
184 : * If a file is present in the disk.
185 : * It's used only in scan to detect present and missing files.
186 : */
187 : #define FILE_IS_PRESENT 0x01
188 :
189 : /**
190 : * If it's an excluded file from the processing.
191 : * It's used in both check and fix to mark files to exclude from the processing.
192 : */
193 : #define FILE_IS_EXCLUDED 0x02
194 :
195 : /**
196 : * If a fix was attempted but it failed.
197 : * It's used only in fix to mark that some data is unrecoverable.
198 : */
199 : #define FILE_IS_DAMAGED 0x04
200 :
201 : /**
202 : * If a fix was done.
203 : * It's used only in fix to mark that some data was recovered.
204 : */
205 : #define FILE_IS_FIXED 0x08
206 :
207 : /**
208 : * If the file was originally missing, and it was created in the fix process.
209 : * It's used only in fix to mark files recovered from scratch,
210 : * meaning that they don't have any previous content.
211 : * This is important because it means that deleting them, you are not going
212 : * to lose something that cannot be recovered.
213 : * Note that excluded files won't ever get this flag.
214 : */
215 : #define FILE_IS_CREATED 0x10
216 :
217 : /**
218 : * If the file has completed its processing, meaning that it won't be opened anymore.
219 : * It's used only in fix to mark when we finish processing one file.
220 : * Note that excluded files won't ever get this flag.
221 : */
222 : #define FILE_IS_FINISHED 0x20
223 :
224 : /**
225 : * If the file hash was obtained from a file copy
226 : * identified by the same name, size and stamp.
227 : */
228 : #define FILE_IS_COPY 0x40
229 :
230 : /**
231 : * If the file was opened.
232 : * It's used in fix to detect if it's the first time a file is opened.
233 : */
234 : #define FILE_IS_OPENED 0x80
235 :
236 : /**
237 : * If the file is modified from the latest sync.
238 : * It's used in fix to store the state of the file before being modified.
239 : */
240 : #define FILE_IS_UNSYNCED 0x100
241 :
242 : /**
243 : * If the file is without inode.
244 : * It could happen in file-systems where inodes are not persistent,
245 : * or when restoring a full disk with "fix".
246 : * In such cases we have to clear any stored duplicate inode.
247 : * After the scan process completes, no file should have this flag set.
248 : */
249 : #define FILE_IS_WITHOUT_INODE 0x200
250 :
251 : /**
252 : * The file is deleted.
253 : * This happens when a file is deleted from the array,
254 : * but it's keep inside the parity until the next sync.
255 : *
256 : * During the file-system check we needs this information,
257 : * because deleted files may be present only partially.
258 : */
259 : #define FILE_IS_DELETED 0x400
260 :
261 : /**
262 : * The file is missing.
263 : * This happens in fix/check when a file is cannot be opened,
264 : * and marking it as such prevents to retry to open it again.
265 : */
266 : #define FILE_IS_MISSING 0x800
267 :
268 : /**
269 : * During scan if the file is found missing and relocated to another place
270 : *
271 : * This is used to avoid to report it "removed".
272 : *
273 : * This specific bit is written using protection from the stamp_mutex during
274 : * the multithread scan.
275 : */
276 : #define FILE_IS_RELOCATED 0x1000
277 :
278 : #define FILE_IS_HARDLINK 0x10000 /**< If it's an hardlink. */
279 : #define FILE_IS_SYMLINK 0x20000 /**< If it's a file symlink. */
280 : #define FILE_IS_SYMDIR 0x40000 /**< If it's a dir symlink for Windows. Not yet supported. */
281 : #define FILE_IS_JUNCTION 0x80000 /**< If it's a junction for Windows. Not yet supported. */
282 : #define FILE_IS_LINK_MASK 0xF0000 /**< Mask for link type. */
283 :
284 : /**
285 : * File.
286 : */
287 : struct snapraid_file {
288 : int64_t mtime_sec; /**< Modification time. */
289 : uint64_t inode; /**< Inode. */
290 : uint64_t physical; /**< Physical offset of the file. */
291 : data_off_t size; /**< Size of the file. */
292 : struct snapraid_block* blockvec; /**< All the blocks of the file. */
293 : int mtime_nsec; /**< Modification time nanoseconds. In the range 0 <= x < 1,000,000,000, or STAT_NSEC_INVALID if not present. */
294 : block_off_t blockmax; /**< Number of blocks. */
295 : unsigned flag; /**< FILE_IS_* flags. */
296 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
297 :
298 : /* nodes for data structures */
299 : tommy_node nodelist;
300 : tommy_hashdyn_node nodeset;
301 : tommy_hashdyn_node pathset;
302 : tommy_hashdyn_node stampset;
303 : };
304 :
305 : /**
306 : * Symbolic Link.
307 : */
308 : struct snapraid_link {
309 : unsigned flag; /**< FILE_IS_* flags. */
310 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
311 : char* linkto; /**< Link to. */
312 :
313 : /* nodes for data structures */
314 : tommy_node nodelist;
315 : tommy_hashdyn_node nodeset;
316 : };
317 :
318 : /**
319 : * Dir.
320 : */
321 : struct snapraid_dir {
322 : unsigned flag; /**< FILE_IS_* flags. */
323 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
324 :
325 : /* nodes for data structures */
326 : tommy_node nodelist;
327 : tommy_hashdyn_node nodeset;
328 : };
329 :
330 : /**
331 : * Chunk.
332 : *
333 : * A extent represents a fragment of a file mapped into the parity.
334 : */
335 : struct snapraid_extent {
336 : struct snapraid_file* file; /**< File containing this extent. */
337 : block_off_t parity_pos; /**< Parity position. */
338 : block_off_t file_pos; /**< Position in the file. */
339 : block_off_t count; /**< Number of sequential blocks in the file and parity. */
340 : tommy_tree_node parity_node; /**< Tree sorted by <parity_pos>. */
341 : tommy_tree_node file_node; /**< Tree sorter by <file,file_pos>. */
342 : };
343 :
344 :
345 : /**
346 : * Other disk.
347 : */
348 : struct snapraid_extra {
349 : char name[PATH_MAX]; /**< Name of the disk. */
350 : char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */
351 : char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
352 : int smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */
353 : char uuid[UUID_MAX]; /**< UUID of the disk. They are probed during the config reading. */
354 : char fstype[FSINFO_MAX]; /**< Filesystem type */
355 : char fslabel[FSINFO_MAX]; /**< Filesystem label */
356 :
357 : uint64_t device; /**< Device identifier. */
358 :
359 : /* nodes for data structures */
360 : tommy_node node;
361 : };
362 :
363 : /**
364 : * Data disk.
365 : */
366 : struct snapraid_disk {
367 : char name[PATH_MAX]; /**< Name of the disk. */
368 : char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */
369 : char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
370 : int smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */
371 : char uuid[UUID_MAX]; /**< UUID of the disk. They are probed during the config reading. */
372 : char fstype[FSINFO_MAX]; /**< Filesystem type */
373 : char fslabel[FSINFO_MAX]; /**< Filesystem label */
374 :
375 : uint64_t device; /**< Device identifier. */
376 : block_off_t total_blocks; /**< Number of total blocks. */
377 : block_off_t free_blocks; /**< Number of free blocks at the last sync. */
378 :
379 : uint64_t tick; /**< Usage time. */
380 : uint64_t progress_tick[PROGRESS_MAX]; /**< Last ticks of progress. */
381 : unsigned cached_blocks; /**< Number of IO blocks cached. */
382 : struct snapraid_file* progress_file; /**< File in progress. */
383 :
384 : /**
385 : * First free searching block.
386 : * Note that it doesn't necessarily point at the first free block,
387 : * but it just tell you that no free block is present before this position.
388 : */
389 : block_off_t first_free_block;
390 :
391 : int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */
392 : int has_volatile_hardlinks; /**< If the underline file-system has not synchronized metadata for hardlink (NTFS). */
393 : int has_unreliable_physical; /**< If the physical offset of files has duplicates. */
394 : int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */
395 : int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */
396 : int had_empty_uuid; /**< If the disk had an empty UUID, meaning that it's a new disk. */
397 : int mapping_idx; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
398 : int skip_access; /**< If the disk is inaccessible and it should be skipped. */
399 :
400 : #if HAVE_THREAD
401 : int single_thread; /**< Running in single thread, and then not needing the mutex */
402 :
403 : /**
404 : * Mutex for protecting the filesystem structure.
405 : *
406 : * Specifically, this protects ::fs_parity, ::fs_file, and ::fs_last,
407 : * meaning that it protects only extents.
408 : *
409 : * Files, links and dirs are not protected as they are not expected to
410 : * change during multithread processing.
411 : */
412 : thread_mutex_t fs_mutex;
413 :
414 : /**
415 : * Mutex for protecting the scan process.
416 : *
417 : * It's used during the scan process to protect the stampset to identify copy of files
418 : * and the FILE_IS_RELOCATED flag of files.
419 : *
420 : * Note that only the FILE_IS_RELOCATED single bit is protected.
421 : * Not the other bits.
422 : */
423 : thread_mutex_t stamp_mutex;
424 : #endif
425 :
426 : /**
427 : * Mapping of extents in the parity.
428 : * Sorted by <parity_pos> and by <file,file_pos>
429 : */
430 : tommy_tree fs_parity;
431 : tommy_tree fs_file;
432 :
433 : /**
434 : * Last extent we accessed.
435 : * It's used to optimize access of sequential blocks.
436 : */
437 : struct snapraid_extent* fs_last;
438 :
439 : /**
440 : * List of all the snapraid_file for the disk.
441 : */
442 : tommy_list filelist;
443 :
444 : /**
445 : * List of all the deleted file for the disk.
446 : *
447 : * These files are kept allocated, because the blocks are still referenced in
448 : * the ::blockarr.
449 : */
450 : tommy_list deletedlist;
451 :
452 : tommy_hashdyn inodeset; /**< Hashtable by inode of all the files. */
453 : tommy_hashdyn pathset; /**< Hashtable by path of all the files. */
454 : tommy_hashdyn stampset; /**< Hashtable by stamp (size and time) of all the files. */
455 : tommy_list linklist; /**< List of all the links. */
456 : tommy_hashdyn linkset; /**< Hashtable by name of all the links. */
457 : tommy_list dirlist; /**< List of all the empty dirs. */
458 : tommy_hashdyn dirset; /**< Hashtable by name of all the empty dirs. */
459 :
460 : /* nodes for data structures */
461 : tommy_node node;
462 : };
463 :
464 : /**
465 : * Disk mapping.
466 : */
467 : struct snapraid_map {
468 : char name[PATH_MAX]; /**< Name of the disk. */
469 : char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. They are read from the content file. */
470 : block_off_t total_blocks; /**< Number of total blocks. */
471 : block_off_t free_blocks; /**< Number of free blocks at last 'sync'. */
472 : unsigned position; /**< Position of the disk in the parity. */
473 :
474 : /* nodes for data structures */
475 : tommy_node node;
476 : };
477 :
478 : /**
479 : * Max number of parity split.
480 : */
481 : #define SPLIT_MAX 8
482 :
483 : /**
484 : * Invalid parity size.
485 : *
486 : * This value is used to identify new parities,
487 : * like when you alter the configuration adding
488 : * a new parity level, creating it with 'fix'.
489 : * Given that 'fix' doesn't write the content file,
490 : * the new size will be written only at the next
491 : * 'sync'.
492 : */
493 : #define PARITY_SIZE_INVALID -1LL
494 :
495 : /**
496 : * Parity split.
497 : */
498 : struct snapraid_split {
499 : char path[PATH_MAX]; /**< Path of the parity file. */
500 : char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. They are probed during the config reading, and later read from the content file. */
501 : char fstype[FSINFO_MAX]; /**< Filesystem type */
502 : char fslabel[FSINFO_MAX]; /**< Filesystem label */
503 :
504 : /**
505 : * Size of the parity split.
506 : * Only the latest not zero size is allowed to grow.
507 : * If the value is unset, it's PARITY_SIZE_INVALID.
508 : */
509 : data_off_t size;
510 :
511 : uint64_t device; /**< Device identifier of the parity. */
512 : };
513 :
514 : /**
515 : * Parity.
516 : */
517 : struct snapraid_parity {
518 : struct snapraid_split split_map[SPLIT_MAX]; /**< Parity splits. */
519 : unsigned split_mac; /**< Number of parity splits. */
520 : char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
521 : int smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */
522 : block_off_t total_blocks; /**< Number of total blocks. */
523 : block_off_t free_blocks; /**< Number of free blocks at the last sync. */
524 : int is_excluded_by_filter; /**< If the parity is excluded by filters. */
525 : int skip_access; /**< If at least one of the parity disk is inaccessible and it should be skipped. */
526 : uint64_t tick; /**< Usage time. */
527 : uint64_t progress_tick[PROGRESS_MAX]; /**< Last cpu ticks of progress. */
528 : unsigned cached_blocks; /**< Number of IO blocks cached. */
529 : };
530 :
531 : /**
532 : * Info.
533 : */
534 : typedef uint64_t snapraid_info;
535 :
536 : /**
537 : * Allocate a content.
538 : */
539 : struct snapraid_content* content_alloc(const char* path, uint64_t dev);
540 :
541 : /**
542 : * Deallocate a content.
543 : */
544 : void content_free(struct snapraid_content* content);
545 :
546 : /**
547 : * Allocate a filter pattern for files and directories.
548 : */
549 : struct snapraid_filter* filter_alloc_file(int is_include, const char* root, const char* pattern);
550 :
551 : /**
552 : * Allocate a filter pattern for disks.
553 : */
554 : struct snapraid_filter* filter_alloc_disk(int is_include, const char* pattern);
555 :
556 : /**
557 : * Deallocate an exclusion.
558 : */
559 : void filter_free(void* filter);
560 :
561 : /**
562 : * Filter type description.
563 : */
564 : const char* filter_type(struct snapraid_filter* filter, char* out, size_t out_size);
565 :
566 : /**
567 : * Filter hidden files.
568 : * Return !=0 if it matches and it should be excluded.
569 : */
570 3335517 : static inline int filter_hidden(int enable, struct dirent* dd)
571 : {
572 3335517 : if (enable && dirent_hidden(dd)) {
573 0 : return 1; /* filter out */
574 : }
575 :
576 3335517 : return 0;
577 : }
578 :
579 : /**
580 : * Filter a path using a list of filters.
581 : * For each element of the path all the filters are applied, until the first one that matches.
582 : * Return !=0 if it should be excluded.
583 : */
584 : int filter_path(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
585 :
586 : /**
587 : * Filter a file/link/dir if missing.
588 : * This call imply a disk check for the file presence.
589 : * Return !=0 if the file is present and it should be excluded.
590 : */
591 : int filter_existence(int filter_missing, const char* dir, const char* sub);
592 :
593 : /**
594 : * Filter a file if bad.
595 : * Return !=0 if the file is correct and it should be excluded.
596 : */
597 : int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file);
598 :
599 : /**
600 : * Filter a dir using a list of filters.
601 : * For each element of the path all the filters are applied, until the first one that matches.
602 : * Thesesdir are always by included by default, to allow to apply rules at the contained files.
603 : * Return !=0 if should be excluded.
604 : */
605 : int filter_subdir(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
606 :
607 : /**
608 : * Filter a dir using a list of filters.
609 : * For each element of the path all the filters are applied, until the first one that matches.
610 : * Return !=0 if should be excluded.
611 : */
612 : int filter_emptydir(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
613 :
614 : /**
615 : * Filter a path if it's a content file.
616 : * Return !=0 if should be excluded.
617 : */
618 : int filter_content(tommy_list* contentlist, const char* path);
619 :
620 : /**
621 : * Check if the specified hash is invalid.
622 : *
623 : * An invalid hash is represented with all bytes at 0x00.
624 : *
625 : * If working with reduced hash lengths, this function always return 0.
626 : */
627 793059 : static inline int hash_is_invalid(const unsigned char* hash)
628 : {
629 : int i;
630 :
631 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
632 793059 : if (BLOCK_HASH_SIZE != HASH_MAX)
633 0 : return 0;
634 :
635 1215649 : for (i = 0; i < BLOCK_HASH_SIZE; ++i)
636 1189431 : if (hash[i] != 0x00)
637 766841 : return 0;
638 :
639 26218 : return 1;
640 : }
641 :
642 : /**
643 : * Check if the specified hash represent the zero block.
644 : *
645 : * A zero hash is represented with all bytes at 0xFF.
646 : *
647 : * If working with reduced hash lengths, this function always return 0.
648 : */
649 796633 : static inline int hash_is_zero(const unsigned char* hash)
650 : {
651 : int i;
652 :
653 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
654 796633 : if (BLOCK_HASH_SIZE != HASH_MAX)
655 0 : return 0;
656 :
657 1064110 : for (i = 0; i < BLOCK_HASH_SIZE; ++i)
658 1047593 : if (hash[i] != 0xFF)
659 780116 : return 0;
660 :
661 16517 : return 1;
662 : }
663 :
664 : /**
665 : * Check if the specified hash is unequivocally representing the data.
666 : *
667 : * If working with reduced hash lengths, this function always return 0.
668 : */
669 44316 : static inline int hash_is_unique(const unsigned char* hash)
670 : {
671 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
672 44316 : if (BLOCK_HASH_SIZE != HASH_MAX)
673 0 : return 0;
674 :
675 44316 : return !hash_is_zero(hash) && !hash_is_invalid(hash);
676 : }
677 :
678 : /**
679 : * Set the hash to the special INVALID value.
680 : */
681 9820268 : static inline void hash_invalid_set(unsigned char* hash)
682 : {
683 9820268 : memset(hash, 0x00, BLOCK_HASH_SIZE);
684 9820268 : }
685 :
686 : /**
687 : * Set the hash to the special ZERO value.
688 : */
689 117596 : static inline void hash_zero_set(unsigned char* hash)
690 : {
691 117596 : memset(hash, 0xFF, BLOCK_HASH_SIZE);
692 117596 : }
693 :
694 : /**
695 : * Allocated space for block.
696 : */
697 72180989 : static inline size_t block_sizeof(void)
698 : {
699 72180989 : return 1 + BLOCK_HASH_SIZE;
700 : }
701 :
702 : /**
703 : * Get the state of the block.
704 : *
705 : * For this function, it's allowed to pass a NULL block
706 : * pointer than results in the BLOCK_STATE_EMPTY state.
707 : */
708 77475741 : static inline unsigned block_state_get(const struct snapraid_block* block)
709 : {
710 77475741 : if (block == BLOCK_NULL)
711 2553373 : return BLOCK_STATE_EMPTY;
712 :
713 74922368 : return block->state;
714 : }
715 :
716 : /**
717 : * Set the state of the block.
718 : */
719 20337038 : static inline void block_state_set(struct snapraid_block* block, unsigned state)
720 : {
721 20337038 : block->state = state;
722 20337038 : }
723 :
724 : /**
725 : * Check if the specified block has an updated hash.
726 : *
727 : * Note that EMPTY / CHG / DELETED return 0.
728 : */
729 2320312 : static inline int block_has_updated_hash(const struct snapraid_block* block)
730 : {
731 2320312 : unsigned state = block_state_get(block);
732 :
733 2320312 : return state == BLOCK_STATE_BLK || state == BLOCK_STATE_REP;
734 : }
735 :
736 : /**
737 : * Check if the specified block has a past hash,
738 : * i.e. the hash of the data that it's now overwritten or lost.
739 : *
740 : * Note that EMPTY / BLK / REP return 0.
741 : */
742 2821726 : static inline int block_has_past_hash(const struct snapraid_block* block)
743 : {
744 2821726 : unsigned state = block_state_get(block);
745 :
746 2821726 : return state == BLOCK_STATE_CHG || state == BLOCK_STATE_DELETED;
747 : }
748 :
749 : /**
750 : * Check if the specified block is part of a file.
751 : *
752 : * Note that EMPTY / DELETED return 0.
753 : */
754 26274546 : static inline int block_has_file(const struct snapraid_block* block)
755 : {
756 26274546 : unsigned state = block_state_get(block);
757 :
758 : return state == BLOCK_STATE_BLK
759 26274546 : || state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
760 : }
761 :
762 : /**
763 : * Check if the block has an invalid parity than needs to be updated.
764 : *
765 : * Note that EMPTY / BLK return 0.
766 : */
767 24126901 : static inline int block_has_invalid_parity(const struct snapraid_block* block)
768 : {
769 24126901 : unsigned state = block_state_get(block);
770 :
771 : return state == BLOCK_STATE_DELETED
772 24126901 : || state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
773 : }
774 :
775 : /**
776 : * Check if the block is part of a file with valid parity.
777 : *
778 : * Note that anything different than BLK return 0.
779 : */
780 223383 : static inline int block_has_file_and_valid_parity(const struct snapraid_block* block)
781 : {
782 223383 : unsigned state = block_state_get(block);
783 :
784 223383 : return state == BLOCK_STATE_BLK;
785 : }
786 :
787 40871544 : static inline int file_flag_has(const struct snapraid_file* file, unsigned mask)
788 : {
789 40871544 : return (file->flag & mask) == mask;
790 : }
791 :
792 6051430 : static inline void file_flag_set(struct snapraid_file* file, unsigned mask)
793 : {
794 6051430 : file->flag |= mask;
795 6051430 : }
796 :
797 18832 : static inline void file_flag_clear(struct snapraid_file* file, unsigned mask)
798 : {
799 18832 : file->flag &= ~mask;
800 18832 : }
801 :
802 : /**
803 : * Allocate a file.
804 : */
805 : struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical);
806 :
807 : /**
808 : * Duplicate a file.
809 : */
810 : struct snapraid_file* file_dup(struct snapraid_file* copy);
811 :
812 : /**
813 : * Deallocate a file.
814 : */
815 : void file_free(struct snapraid_file* file);
816 :
817 : /**
818 : * Rename a file.
819 : */
820 : void file_rename(struct snapraid_file* file, const char* sub);
821 :
822 : /**
823 : * Copy a file.
824 : */
825 : void file_copy(struct snapraid_file* src_file, struct snapraid_file* dest_file);
826 :
827 : /**
828 : * Return the block at the specified position.
829 : *
830 : * Note that the block size if a runtime value.
831 : */
832 68234494 : static inline struct snapraid_block* file_block(struct snapraid_file* file, size_t pos)
833 : {
834 68234494 : unsigned char* ptr = (unsigned char*)file->blockvec;
835 :
836 68234494 : return (struct snapraid_block*)(ptr + pos * block_sizeof());
837 : }
838 :
839 : /**
840 : * Return the name of the file, without the dir.
841 : */
842 : const char* file_name(const struct snapraid_file* file);
843 :
844 : /**
845 : * Check if the block is the last in the file.
846 : */
847 : int file_block_is_last(struct snapraid_file* file, block_off_t file_pos);
848 :
849 : /**
850 : * Get the size in bytes of the block.
851 : * If it's the last block of a file it could be less than block_size.
852 : */
853 : unsigned file_block_size(struct snapraid_file* file, block_off_t file_pos, unsigned block_size);
854 :
855 : /**
856 : * Compare a file with an inode.
857 : */
858 : int file_inode_compare_to_arg(const void* void_arg, const void* void_data);
859 :
860 : /**
861 : * Compare files by inode.
862 : */
863 : int file_inode_compare(const void* void_a, const void* void_b);
864 :
865 : /**
866 : * Compare files by path.
867 : */
868 : int file_path_compare(const void* void_a, const void* void_b);
869 :
870 : /**
871 : * Compare files by physical address.
872 : */
873 : int file_physical_compare(const void* void_a, const void* void_b);
874 :
875 : /**
876 : * Compute the hash of a file inode.
877 : */
878 5416782 : static inline tommy_uint32_t file_inode_hash(uint64_t inode)
879 : {
880 5416782 : return (tommy_uint32_t)tommy_inthash_u64(inode);
881 : }
882 :
883 : /**
884 : * Compare a file with a path.
885 : */
886 : int file_path_compare_to_arg(const void* void_arg, const void* void_data);
887 :
888 : /**
889 : * Compare a file with another file for name, stamp, and both.
890 : */
891 : int file_name_compare(const void* void_a, const void* void_b);
892 : int file_stamp_compare(const void* void_a, const void* void_b);
893 : int file_namestamp_compare(const void* void_a, const void* void_b);
894 : int file_pathstamp_compare(const void* void_a, const void* void_b);
895 :
896 : /**
897 : * Compute the hash of a file path.
898 : */
899 4130215 : static inline tommy_uint32_t file_path_hash(const char* sub)
900 : {
901 4130215 : return tommy_hash_u32(0, sub, strlen(sub));
902 : }
903 :
904 : /**
905 : * Compute the hash of a file stamp.
906 : */
907 6590541 : static inline tommy_uint32_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec)
908 : {
909 6590541 : return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32(mtime_sec ^ tommy_inthash_u32(mtime_nsec)));
910 : }
911 :
912 : /**
913 : * Allocate a extent.
914 : */
915 : struct snapraid_extent* extent_alloc(block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos, block_off_t count);
916 :
917 : /**
918 : * Deallocate a extent.
919 : */
920 : void extent_free(struct snapraid_extent* extent);
921 :
922 : /**
923 : * Compare extent by parity position.
924 : */
925 : int extent_parity_compare(const void* void_a, const void* void_b);
926 :
927 : /**
928 : * Compare extent by file and file position.
929 : */
930 : int extent_file_compare(const void* void_a, const void* void_b);
931 :
932 223705 : static inline int link_flag_has(const struct snapraid_link* slink, unsigned mask)
933 : {
934 223705 : return (slink->flag & mask) == mask;
935 : }
936 :
937 43664 : static inline void link_flag_set(struct snapraid_link* slink, unsigned mask)
938 : {
939 43664 : slink->flag |= mask;
940 43664 : }
941 :
942 : static inline void link_flag_clear(struct snapraid_link* slink, unsigned mask)
943 : {
944 : slink->flag &= ~mask;
945 : }
946 :
947 7 : static inline void link_flag_let(struct snapraid_link* slink, unsigned flag, unsigned mask)
948 : {
949 7 : slink->flag &= ~mask;
950 7 : slink->flag |= flag & mask;
951 7 : }
952 :
953 99564 : static inline unsigned link_flag_get(struct snapraid_link* slink, unsigned mask)
954 : {
955 99564 : return slink->flag & mask;
956 : }
957 :
958 : /**
959 : * Allocate a link.
960 : */
961 : struct snapraid_link* link_alloc(const char* name, const char* slink, unsigned link_flag);
962 :
963 : /**
964 : * Deallocate a link.
965 : */
966 : void link_free(struct snapraid_link* slink);
967 :
968 : /**
969 : * Compare a link with a name.
970 : */
971 : int link_name_compare_to_arg(const void* void_arg, const void* void_data);
972 :
973 : /**
974 : * Compare links by path.
975 : */
976 : int link_alpha_compare(const void* void_a, const void* void_b);
977 :
978 : /**
979 : * Compute the hash of a link name.
980 : */
981 166594 : static inline tommy_uint32_t link_name_hash(const char* name)
982 : {
983 166594 : return tommy_hash_u32(0, name, strlen(name));
984 : }
985 :
986 1028 : static inline int dir_flag_has(const struct snapraid_dir* dir, unsigned mask)
987 : {
988 1028 : return (dir->flag & mask) == mask;
989 : }
990 :
991 318 : static inline void dir_flag_set(struct snapraid_dir* dir, unsigned mask)
992 : {
993 318 : dir->flag |= mask;
994 318 : }
995 :
996 : static inline void dir_flag_clear(struct snapraid_dir* dir, unsigned mask)
997 : {
998 : dir->flag &= ~mask;
999 : }
1000 :
1001 : /**
1002 : * Allocate a dir.
1003 : */
1004 : struct snapraid_dir* dir_alloc(const char* name);
1005 :
1006 : /**
1007 : * Deallocate a dir.
1008 : */
1009 : void dir_free(struct snapraid_dir* dir);
1010 :
1011 : /**
1012 : * Compare a dir with a name.
1013 : */
1014 : int dir_name_compare(const void* void_arg, const void* void_data);
1015 :
1016 : /**
1017 : * Compute the hash of a dir name.
1018 : */
1019 1167 : static inline tommy_uint32_t dir_name_hash(const char* name)
1020 : {
1021 1167 : return tommy_hash_u32(0, name, strlen(name));
1022 : }
1023 :
1024 : /**
1025 : * Allocate a disk.
1026 : */
1027 : struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid, int skip_access);
1028 :
1029 : /**
1030 : * Deallocate a disk.
1031 : */
1032 : void disk_free(struct snapraid_disk* disk);
1033 :
1034 : /**
1035 : * Allocate a extra disk.
1036 : */
1037 : struct snapraid_extra* extra_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid);
1038 :
1039 : /**
1040 : * Deallocate a other disk.
1041 : */
1042 : void extra_free(struct snapraid_extra* other);
1043 :
1044 : /**
1045 : * Get the size of the disk in blocks.
1046 : */
1047 : block_off_t fs_size(struct snapraid_disk* disk);
1048 :
1049 : /**
1050 : * Check if a disk is totally empty and can be discarded from the content file.
1051 : * A disk is empty if it doesn't contain any file, symlink, hardlink or dir
1052 : * and without any DELETED block.
1053 : * The blockmax is used to limit the search of DELETED block up to blockmax.
1054 : */
1055 : int fs_is_empty(struct snapraid_disk* disk, block_off_t blockmax);
1056 :
1057 : /**
1058 : * Check the file-system for errors.
1059 : * Return 0 if it's OK.
1060 : */
1061 : int fs_check(struct snapraid_disk* disk);
1062 :
1063 : /**
1064 : * Allocate a parity position for the specified file position.
1065 : *
1066 : * After this call you can use the par2file/par2block operations
1067 : * to query the relation.
1068 : *
1069 : * \note This function is NOT thread-safe as it uses the disk cache. +
1070 : */
1071 : void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos);
1072 :
1073 : /**
1074 : * Deallocate the parity position.
1075 : *
1076 : * After this call the par2file/par2block operations
1077 : * won't find anymore the parity association.
1078 : *
1079 : * \note This function is NOT thread-safe as it uses the disk cache.
1080 : */
1081 : void fs_deallocate(struct snapraid_disk* disk, block_off_t pos);
1082 :
1083 : /**
1084 : * Get the block from the file position.
1085 : */
1086 : struct snapraid_block* fs_file2block_get(struct snapraid_file* file, block_off_t file_pos);
1087 :
1088 : /**
1089 : * Get the file position from the parity position.
1090 : * Return 0 if no file is using it.
1091 : */
1092 : struct snapraid_file* fs_par2file_find(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos);
1093 :
1094 : /**
1095 : * Get the file position from the parity position.
1096 : */
1097 11875406 : static inline struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos)
1098 : {
1099 : struct snapraid_file* ret;
1100 :
1101 11875406 : ret = fs_par2file_find(disk, parity_pos, file_pos);
1102 11875406 : if (ret == 0) {
1103 : /* LCOV_EXCL_START */
1104 : log_fatal(EINTERNAL, "Internal inconsistency: Deresolving parity to file at position '%u' in disk '%s'\n", parity_pos, disk->name);
1105 : os_abort();
1106 : /* LCOV_EXCL_STOP */
1107 : }
1108 :
1109 11875406 : return ret;
1110 : }
1111 :
1112 : /**
1113 : * Get the parity position from the file position.
1114 : * Return POS_NULL if no parity is allocated.
1115 : */
1116 : block_off_t fs_file2par_find(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos);
1117 :
1118 : /**
1119 : * Get the parity position from the file position.
1120 : */
1121 5093355 : static inline block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
1122 : {
1123 : block_off_t ret;
1124 :
1125 5093355 : ret = fs_file2par_find(disk, file, file_pos);
1126 5093355 : if (ret == POS_NULL) {
1127 : /* LCOV_EXCL_START */
1128 : log_fatal(EINTERNAL, "Internal inconsistency: Resolving file '%s' at position '%u/%u' in disk '%s'\n", file->sub, file_pos, file->blockmax, disk->name);
1129 : os_abort();
1130 : /* LCOV_EXCL_STOP */
1131 : }
1132 :
1133 5093355 : return ret;
1134 : }
1135 :
1136 : /**
1137 : * Get the block from the parity position.
1138 : * Return BLOCK_NULL==0 if the block is over the end of the disk or not used.
1139 : */
1140 : struct snapraid_block* fs_par2block_find(struct snapraid_disk* disk, block_off_t parity_pos);
1141 :
1142 : /**
1143 : * Get the block from the parity position.
1144 : */
1145 17742 : static inline struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos)
1146 : {
1147 : struct snapraid_block* ret;
1148 :
1149 17742 : ret = fs_par2block_find(disk, parity_pos);
1150 17742 : if (ret == BLOCK_NULL) {
1151 : /* LCOV_EXCL_START */
1152 : log_fatal(EINTERNAL, "Internal inconsistency: Deresolving parity to block at position '%u' in disk '%s'\n", parity_pos, disk->name);
1153 : os_abort();
1154 : /* LCOV_EXCL_STOP */
1155 : }
1156 :
1157 17742 : return ret;
1158 : }
1159 :
1160 : /**
1161 : * Allocate a disk mapping.
1162 : * Uses uuid="" if not available.
1163 : */
1164 : struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid);
1165 :
1166 : /**
1167 : * Deallocate a disk mapping.
1168 : */
1169 : void map_free(struct snapraid_map* map);
1170 :
1171 : /**
1172 : * Mask used to store additional information in the info bits.
1173 : *
1174 : * These bits reduce the granularity of the time in the memory representation.
1175 : */
1176 : #define INFO_MASK ((snapraid_info)0x7)
1177 :
1178 : /**
1179 : * Make an info.
1180 : */
1181 191500 : static inline snapraid_info info_make(time_t last_access, int error, int rehash, int justsynced)
1182 : {
1183 : /* clear the lowest bits as reserved for other information */
1184 191500 : snapraid_info info = last_access & ~INFO_MASK;
1185 :
1186 191500 : if (error != 0)
1187 5301 : info |= 0x1;
1188 191500 : if (rehash != 0)
1189 14266 : info |= 0x2;
1190 191500 : if (justsynced != 0)
1191 145259 : info |= 0x4;
1192 191500 : return info;
1193 : }
1194 :
1195 : /**
1196 : * Extract the time information.
1197 : * This is the last time when the block was know to be correct.
1198 : * The "scrubbed" info tells if the time is referring at the latest sync or scrub.
1199 : */
1200 818895 : static inline time_t info_get_time(snapraid_info info)
1201 : {
1202 818895 : return info & ~INFO_MASK;
1203 : }
1204 :
1205 : /**
1206 : * Extract the error information.
1207 : * Report if the block address had some problem.
1208 : */
1209 253312 : static inline int info_get_bad(snapraid_info info)
1210 : {
1211 253312 : return (info & 0x1) != 0;
1212 : }
1213 :
1214 : /**
1215 : * Extract the rehash information.
1216 : * Report if the block address is using the old hash and needs to be rehashed.
1217 : */
1218 2106387 : static inline int info_get_rehash(snapraid_info info)
1219 : {
1220 2106387 : return (info & 0x2) != 0;
1221 : }
1222 :
1223 : /**
1224 : * Extract the scrubbed information.
1225 : * Report if the block address was never scrubbed.
1226 : */
1227 19146 : static inline int info_get_justsynced(snapraid_info info)
1228 : {
1229 19146 : return (info & 0x4) != 0;
1230 : }
1231 :
1232 : /**
1233 : * Mark the block address as with error.
1234 : */
1235 1772 : static inline snapraid_info info_set_bad(snapraid_info info)
1236 : {
1237 1772 : return info | 0x1;
1238 : }
1239 :
1240 : /**
1241 : * Mark the block address as with rehash.
1242 : */
1243 9275 : static inline snapraid_info info_set_rehash(snapraid_info info)
1244 : {
1245 9275 : return info | 0x2;
1246 : }
1247 :
1248 : /**
1249 : * Set the info at the specified position.
1250 : * The position is allocated if not yet done.
1251 : */
1252 1822675 : static inline void info_set(tommy_arrayblkof* array, block_off_t pos, snapraid_info info)
1253 : {
1254 1822675 : tommy_arrayblkof_grow(array, pos + 1);
1255 :
1256 1822675 : memcpy(tommy_arrayblkof_ref(array, pos), &info, sizeof(snapraid_info));
1257 1822675 : }
1258 :
1259 : /**
1260 : * Get the info at the specified position.
1261 : * For not allocated position, 0 is returned.
1262 : */
1263 3253108 : static inline snapraid_info info_get(tommy_arrayblkof* array, block_off_t pos)
1264 : {
1265 : snapraid_info info;
1266 :
1267 3253108 : if (pos >= tommy_arrayblkof_size(array))
1268 264646 : return 0;
1269 :
1270 2988462 : memcpy(&info, tommy_arrayblkof_ref(array, pos), sizeof(snapraid_info));
1271 :
1272 2988462 : return info;
1273 : }
1274 :
1275 : /****************************************************************************/
1276 : /* bucket */
1277 :
1278 : struct snapraid_bucket {
1279 : time_t time_at; /**< Time of the scrub */
1280 : block_off_t count_scrubbed; /**< Number of blocks scrubbed */
1281 : block_off_t count_justsynced; /**< Number of blocks justsynced */
1282 : tommy_node node;
1283 : };
1284 :
1285 : void bucket_free(struct snapraid_bucket* bucket);
1286 :
1287 : /**
1288 : * Insert an bucket record into the hashtable
1289 : */
1290 : void bucket_insert(tommy_hashdyn* bucket_hash, time_t time_at, block_off_t count, int justsynced);
1291 :
1292 : /**
1293 : * Convert the bucket hashtable to a sorted list
1294 : */
1295 : void bucket_to_list(tommy_hashdyn* bucket_hash, tommy_list* bucket_list, block_off_t* bucketcount);
1296 :
1297 : /****************************************************************************/
1298 : /* format */
1299 :
1300 : #define FMT_FILE 0 /**< Print only the file. */
1301 : #define FMT_DISK 1 /**< Print the disk name and the file. */
1302 : #define FMT_PATH 2 /**< Print the full path. */
1303 :
1304 : extern int FMT_MODE;
1305 :
1306 : /**
1307 : * Format a file path for poll reference
1308 : */
1309 : const char* fmt_poll(const struct snapraid_disk* disk, const char* str, char* buffer);
1310 :
1311 : /**
1312 : * Format a path name for terminal reference
1313 : */
1314 : const char* fmt_term(const struct snapraid_disk* disk, const char* str, char* buffer);
1315 :
1316 : /**
1317 : * Format a size adding suffix K/M/G/T
1318 : */
1319 : const char* fmt_size(uint64_t size, char* buffer, size_t buffer_size);
1320 :
1321 : #endif
1322 :
|