Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "elem.h"
21 : #include "import.h"
22 : #include "search.h"
23 : #include "state.h"
24 : #include "support.h"
25 : #include "parity.h"
26 : #include "stream.h"
27 : #include "handle.h"
28 : #include "io.h"
29 : #include "raid/raid.h"
30 : #include "raid/cpu.h"
31 :
32 : /**
33 : * Configure the multithread support.
34 : *
35 : * Multi thread for write could be either faster or slower, depending
36 : * on the specific conditions. With multithreads it's likely faster
37 : * writing to disk, but you'll need to access multiple times the same data,
38 : * being potentially slower.
39 : *
40 : * For upcoming SnapRAID version it's planned to add a mutex protection
41 : * at the file-system structure, slowing down multiple data access,
42 : * so we disable it.
43 : *
44 : * Multi thread for verify is instead always generally faster,
45 : * so we enable it if possible.
46 : */
47 : #if HAVE_PTHREAD
48 : /* #define HAVE_MT_WRITE 1 */
49 : #define HAVE_MT_VERIFY 1
50 : #endif
51 :
52 7 : const char* lev_name(unsigned l)
53 : {
54 7 : switch (l) {
55 2 : case 0 : return "Parity";
56 1 : case 1 : return "2-Parity";
57 1 : case 2 : return "3-Parity";
58 1 : case 3 : return "4-Parity";
59 1 : case 4 : return "5-Parity";
60 1 : case 5 : return "6-Parity";
61 : }
62 :
63 0 : return 0;
64 : }
65 :
66 186606 : const char* lev_config_name(unsigned l)
67 : {
68 186606 : switch (l) {
69 86864 : case 0 : return "parity";
70 60630 : case 1 : return "2-parity";
71 9820 : case 2 : return "3-parity";
72 9793 : case 3 : return "4-parity";
73 9766 : case 4 : return "5-parity";
74 9733 : case 5 : return "6-parity";
75 : }
76 :
77 0 : return 0;
78 : }
79 :
80 6345 : static int lev_config_scan(const char* s, unsigned* level, unsigned* mode)
81 : {
82 6345 : if (strcmp(s, "parity") == 0 || strcmp(s, "1-parity") == 0) {
83 449 : *level = 0;
84 449 : return 0;
85 : }
86 :
87 5896 : if (strcmp(s, "q-parity") == 0 || strcmp(s, "2-parity") == 0) {
88 225 : *level = 1;
89 225 : return 0;
90 : }
91 :
92 5671 : if (strcmp(s, "r-parity") == 0 || strcmp(s, "3-parity") == 0) {
93 200 : *level = 2;
94 200 : return 0;
95 : }
96 :
97 5471 : if (strcmp(s, "4-parity") == 0) {
98 197 : *level = 3;
99 197 : return 0;
100 : }
101 :
102 5274 : if (strcmp(s, "5-parity") == 0) {
103 194 : *level = 4;
104 194 : return 0;
105 : }
106 :
107 5080 : if (strcmp(s, "6-parity") == 0) {
108 190 : *level = 5;
109 190 : return 0;
110 : }
111 :
112 4890 : if (strcmp(s, "z-parity") == 0) {
113 0 : *level = 2;
114 0 : if (mode)
115 0 : *mode = RAID_MODE_VANDERMONDE;
116 0 : return 0;
117 : }
118 :
119 4890 : return -1;
120 : }
121 :
122 265 : const char* lev_raid_name(unsigned mode, unsigned n)
123 : {
124 265 : switch (n) {
125 41 : case 1 : return "par1";
126 25 : case 2 : return "par2";
127 3 : case 3 : if (mode == RAID_MODE_CAUCHY)
128 3 : return "par3";
129 : else
130 0 : return "parz";
131 3 : case 4 : return "par4";
132 4 : case 5 : return "par5";
133 189 : case 6 : return "par6";
134 : }
135 :
136 0 : return 0;
137 : }
138 :
139 267 : void state_init(struct snapraid_state* state)
140 : {
141 : unsigned l, s;
142 :
143 267 : memset(&state->opt, 0, sizeof(state->opt));
144 267 : state->filter_hidden = 0;
145 267 : state->autosave = 0;
146 267 : state->need_write = 0;
147 267 : state->checked_read = 0;
148 267 : state->block_size = 256 * KIBI; /* default 256 KiB */
149 267 : state->raid_mode = RAID_MODE_CAUCHY;
150 267 : state->file_mode = ADVISE_DEFAULT;
151 1869 : for (l = 0; l < LEV_MAX; ++l) {
152 1602 : state->parity[l].split_mac = 0;
153 14418 : for (s = 0; s < SPLIT_MAX; ++s) {
154 12816 : state->parity[l].split_map[s].path[0] = 0;
155 12816 : state->parity[l].split_map[s].uuid[0] = 0;
156 12816 : state->parity[l].split_map[s].size = PARITY_SIZE_INVALID;
157 12816 : state->parity[l].split_map[s].device = 0;
158 : }
159 1602 : state->parity[l].smartctl[0] = 0;
160 1602 : state->parity[l].total_blocks = 0;
161 1602 : state->parity[l].free_blocks = 0;
162 1602 : state->parity[l].skip_access = 0;
163 1602 : state->parity[l].tick = 0;
164 1602 : state->parity[l].cached = 0;
165 1602 : state->parity[l].is_excluded_by_filter = 0;
166 : }
167 267 : state->tick_io = 0;
168 267 : state->tick_misc = 0;
169 267 : state->tick_sched = 0;
170 267 : state->tick_raid = 0;
171 267 : state->tick_hash = 0;
172 267 : state->tick_last = tick();
173 267 : state->share[0] = 0;
174 267 : state->pool[0] = 0;
175 267 : state->pool_device = 0;
176 267 : state->lockfile[0] = 0;
177 267 : state->level = 1; /* default is the lowest protection */
178 267 : state->clear_past_hash = 0;
179 267 : state->no_conf = 0;
180 :
181 267 : tommy_list_init(&state->disklist);
182 267 : tommy_list_init(&state->maplist);
183 267 : tommy_list_init(&state->contentlist);
184 267 : tommy_list_init(&state->filterlist);
185 267 : tommy_list_init(&state->importlist);
186 267 : tommy_hashdyn_init(&state->importset);
187 267 : tommy_hashdyn_init(&state->previmportset);
188 267 : tommy_hashdyn_init(&state->searchset);
189 267 : tommy_arrayblkof_init(&state->infoarr, sizeof(snapraid_info));
190 267 : }
191 :
192 244 : void state_done(struct snapraid_state* state)
193 : {
194 244 : tommy_list_foreach(&state->disklist, (tommy_foreach_func*)disk_free);
195 244 : tommy_list_foreach(&state->maplist, (tommy_foreach_func*)map_free);
196 244 : tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
197 244 : tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
198 244 : tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
199 244 : tommy_hashdyn_foreach(&state->searchset, (tommy_foreach_func*)search_file_free);
200 244 : tommy_hashdyn_done(&state->importset);
201 244 : tommy_hashdyn_done(&state->previmportset);
202 244 : tommy_hashdyn_done(&state->searchset);
203 244 : tommy_arrayblkof_done(&state->infoarr);
204 244 : }
205 :
206 : /**
207 : * Check the configuration.
208 : */
209 265 : static void state_config_check(struct snapraid_state* state, const char* path, tommy_list* filterlist_disk)
210 : {
211 : tommy_node* i;
212 : unsigned l, s;
213 :
214 : /* check for parity level */
215 265 : if (state->raid_mode == RAID_MODE_VANDERMONDE) {
216 0 : if (state->level > 3) {
217 : /* LCOV_EXCL_START */
218 : log_fatal("If you use the z-parity you cannot have more than 3 parities.\n");
219 : exit(EXIT_FAILURE);
220 : /* LCOV_EXCL_STIO */
221 : }
222 : }
223 :
224 : for (l = 0; l < state->level; ++l) {
225 : if (state->parity[l].split_mac == 0) {
226 : /* LCOV_EXCL_START */
227 : log_fatal("No '%s' specification in '%s'\n", lev_config_name(l), path);
228 : exit(EXIT_FAILURE);
229 : /* LCOV_EXCL_STOP */
230 : }
231 : }
232 :
233 265 : if (tommy_list_empty(&state->contentlist)) {
234 : /* LCOV_EXCL_START */
235 : log_fatal("No 'content' specification in '%s'\n", path);
236 : exit(EXIT_FAILURE);
237 : /* LCOV_EXCL_STOP */
238 : }
239 :
240 : /* check for equal paths */
241 1796 : for (i = state->contentlist; i != 0; i = i->next) {
242 1531 : struct snapraid_content* content = i->data;
243 :
244 9917 : for (l = 0; l < state->level; ++l) {
245 41930 : for (s = 0; s < state->parity[l].split_mac; ++s) {
246 33544 : if (pathcmp(state->parity[l].split_map[s].path, content->content) == 0) {
247 : /* LCOV_EXCL_START */
248 : log_fatal("Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
249 : exit(EXIT_FAILURE);
250 : /* LCOV_EXCL_STOP */
251 : }
252 : }
253 : }
254 : }
255 :
256 : /* check device of data disks */
257 265 : if (!state->opt.skip_device && !state->opt.skip_disk_access) {
258 0 : for (i = state->disklist; i != 0; i = i->next) {
259 : tommy_node* j;
260 0 : struct snapraid_disk* disk = i->data;
261 :
262 : /* skip data disks that are not accessible */
263 0 : if (disk->skip_access)
264 0 : continue;
265 :
266 : #ifdef _WIN32
267 : if (disk->device == 0) {
268 : /* LCOV_EXCL_START */
269 : log_fatal("Disk '%s' has a zero serial number.\n", disk->dir);
270 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
271 : log_fatal("it's better to change the serial number of the disk.\n");
272 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
273 : exit(EXIT_FAILURE);
274 : /* LCOV_EXCL_STOP */
275 : }
276 : #endif
277 :
278 0 : for (j = i->next; j != 0; j = j->next) {
279 0 : struct snapraid_disk* other = j->data;
280 0 : if (disk->device == other->device) {
281 0 : if (state->opt.force_device) {
282 : /* note tha we just ignore the issue */
283 : /* and we DON'T mark the disk to be skipped */
284 : /* because we want to use these disks */
285 0 : if (!state->opt.no_warnings)
286 0 : log_fatal("DANGER! Ignoring that disks '%s' and '%s' are on the same device\n", disk->name, other->name);
287 : } else {
288 : /* LCOV_EXCL_START */
289 : log_fatal("Disks '%s' and '%s' are on the same device.\n", disk->dir, other->dir);
290 : #ifdef _WIN32
291 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
292 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
293 : log_fatal("to change one of the disk serial.\n");
294 : #endif
295 : /* in "fix" we allow to continue anyway */
296 : if (strcmp(state->command, "fix") == 0) {
297 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
298 : }
299 : exit(EXIT_FAILURE);
300 : /* LCOV_EXCL_STOP */
301 : }
302 : }
303 : }
304 :
305 : /* skip data disks that are not accessible */
306 0 : if (disk->skip_access)
307 0 : continue;
308 :
309 0 : if (!state->opt.skip_parity_access) {
310 0 : for (l = 0; l < state->level; ++l) {
311 0 : for (s = 0; s < state->parity[l].split_mac; ++s) {
312 0 : if (disk->device == state->parity[l].split_map[s].device) {
313 : /* LCOV_EXCL_START */
314 : log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
315 : #ifdef _WIN32
316 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
317 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
318 : log_fatal("to change one of the disk serial.\n");
319 : #endif
320 : exit(EXIT_FAILURE);
321 : /* LCOV_EXCL_STOP */
322 : }
323 : }
324 : }
325 : }
326 :
327 0 : if (state->pool[0] != 0 && disk->device == state->pool_device) {
328 : /* LCOV_EXCL_START */
329 : log_fatal("Disk '%s' and pool '%s' are on the same device.\n", disk->dir, state->pool);
330 : #ifdef _WIN32
331 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
332 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
333 : log_fatal("to change one of the disk serial.\n");
334 : #endif
335 : exit(EXIT_FAILURE);
336 : /* LCOV_EXCL_STOP */
337 : }
338 : }
339 : }
340 :
341 : /* check device of parity disks */
342 265 : if (!state->opt.skip_device && !state->opt.skip_parity_access) {
343 0 : for (l = 0; l < state->level; ++l) {
344 0 : for (s = 0; s < state->parity[l].split_mac; ++s) {
345 : unsigned j,t;
346 :
347 : /* skip parity disks that are not accessible */
348 0 : if (state->parity[l].skip_access)
349 0 : continue;
350 :
351 : #ifdef _WIN32
352 : if (state->parity[l].split_map[s].device == 0) {
353 : /* LCOV_EXCL_START */
354 : log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].split_map[s].path);
355 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
356 : log_fatal("it's better to change the serial number of the disk.\n");
357 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
358 : exit(EXIT_FAILURE);
359 : /* LCOV_EXCL_STOP */
360 : }
361 : #endif
362 :
363 0 : for (j = l + 1; j < state->level; ++j) {
364 0 : for (t = 0; t < state->parity[j].split_mac; ++t) {
365 0 : if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
366 0 : if (state->opt.force_device) {
367 : /* note tha we just ignore the issue */
368 : /* and we DON'T mark the disk to be skipped */
369 : /* because we want to use these disks */
370 0 : if (!state->opt.no_warnings)
371 0 : log_fatal("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
372 : } else {
373 : /* LCOV_EXCL_START */
374 : log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].split_map[s].path, state->parity[j].split_map[t].path);
375 : #ifdef _WIN32
376 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].split_map[s].device);
377 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
378 : log_fatal("to change one of the disk serial.\n");
379 : #endif
380 : /* in "fix" we allow to continue anyway */
381 : if (strcmp(state->command, "fix") == 0) {
382 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
383 : }
384 : exit(EXIT_FAILURE);
385 : /* LCOV_EXCL_STOP */
386 : }
387 : }
388 : }
389 : }
390 :
391 0 : if (state->pool[0] != 0 && state->pool_device == state->parity[l].split_map[s].device) {
392 : /* LCOV_EXCL_START */
393 : log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].split_map[s].path);
394 : #ifdef _WIN32
395 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
396 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
397 : log_fatal("to change one of the disk serial.\n");
398 : #endif
399 : exit(EXIT_FAILURE);
400 : /* LCOV_EXCL_STOP */
401 : }
402 : }
403 : }
404 : }
405 :
406 : /* check device of pool disk */
407 : #ifdef _WIN32
408 : if (!state->opt.skip_device) {
409 : if (state->pool[0] != 0 && state->pool_device == 0) {
410 : /* LCOV_EXCL_START */
411 : log_fatal("Disk '%s' has a zero serial number.\n", state->pool);
412 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
413 : log_fatal("it's better to change the serial number of the disk.\n");
414 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
415 : exit(EXIT_FAILURE);
416 : /* LCOV_EXCL_STOP */
417 : }
418 : }
419 : #endif
420 :
421 : /* count the content files */
422 265 : if (!state->opt.skip_device && !state->opt.skip_content_access) {
423 : unsigned content_count;
424 :
425 0 : content_count = 0;
426 0 : for (i = state->contentlist; i != 0; i = i->next) {
427 : tommy_node* j;
428 0 : struct snapraid_content* content = i->data;
429 :
430 : /* check if there are others in the same disk */
431 0 : for (j = i->next; j != 0; j = j->next) {
432 0 : struct snapraid_content* other = j->data;
433 0 : if (content->device == other->device) {
434 0 : break;
435 : }
436 : }
437 0 : if (j != 0) {
438 : /* skip it */
439 0 : continue;
440 : }
441 :
442 0 : ++content_count;
443 : }
444 :
445 0 : if (content_count < state->level + 1) {
446 : /* LCOV_EXCL_START */
447 : log_fatal("You must have at least %d 'content' files in different disks.\n", state->level + 1);
448 : exit(EXIT_FAILURE);
449 : /* LCOV_EXCL_STOP */
450 : }
451 : }
452 :
453 : /* check for speed */
454 : #ifdef CONFIG_X86
455 265 : if (!raid_cpu_has_ssse3())
456 : #endif
457 0 : if (state->raid_mode == RAID_MODE_CAUCHY && !state->opt.no_warnings) {
458 0 : if (state->level == 3) {
459 0 : log_fatal("WARNING! Your CPU doesn't have a fast implementation for triple parity.\n");
460 0 : log_fatal("WARNING! It's recommended to switch to 'z-parity' instead than '3-parity'.\n");
461 0 : } else if (state->level > 3) {
462 0 : log_fatal("WARNING! Your CPU doesn't have a fast implementation beyond triple parity.\n");
463 0 : log_fatal("WARNING! It's recommended to reduce the parity levels to triple parity.\n");
464 : }
465 : }
466 :
467 : /* ensure that specified filter disks are valid ones */
468 275 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
469 : tommy_node* j;
470 10 : struct snapraid_filter* filter = i->data;
471 51 : for (j = state->disklist; j != 0; j = j->next) {
472 45 : struct snapraid_disk* disk = j->data;
473 45 : if (fnmatch(filter->pattern, disk->name, FNM_CASEINSENSITIVE_FOR_WIN) == 0)
474 4 : break;
475 : }
476 10 : if (j == 0) {
477 : /* check matching with parity disks */
478 21 : for (l = 0; l < state->level; ++l)
479 21 : if (fnmatch(filter->pattern, lev_config_name(l), FNM_CASEINSENSITIVE_FOR_WIN) == 0)
480 6 : break;
481 6 : if (l == state->level) {
482 : /* LCOV_EXCL_START */
483 : log_fatal("Option -d, --filter-disk %s doesn't match any data or parity disk.\n", filter->pattern);
484 : exit(EXIT_FAILURE);
485 : /* LCOV_EXCL_STOP */
486 : }
487 : }
488 : }
489 265 : }
490 :
491 : /**
492 : * Validate the smartctl command.
493 : *
494 : * It must contains only one %s string, and not other % chars.
495 : */
496 366 : static int validate_smartctl(const char* custom)
497 : {
498 366 : const char* s = custom;
499 366 : int arg = 0;
500 :
501 2562 : while (*s) {
502 1830 : if (s[0] == '%' && s[1] == 's') {
503 183 : if (arg) {
504 : /* LCOV_EXCL_START */
505 : return -1;
506 : /* LCOV_EXCL_STOP */
507 : }
508 183 : arg = 1;
509 1647 : } else if (s[0] == '%') {
510 : /* LCOV_EXCL_START */
511 : return -1;
512 : /* LCOV_EXCL_STOP */
513 : }
514 :
515 1830 : ++s;
516 : }
517 :
518 366 : return 0;
519 : }
520 :
521 266 : void state_config(struct snapraid_state* state, const char* path, const char* command, struct snapraid_option* opt, tommy_list* filterlist_disk)
522 : {
523 : STREAM* f;
524 : unsigned line;
525 : tommy_node* i;
526 : unsigned l, s;
527 :
528 : /* copy the options */
529 266 : state->opt = *opt;
530 :
531 : /* if unset, sort by physical order */
532 266 : if (!state->opt.force_order)
533 1 : state->opt.force_order = SORT_PHYSICAL;
534 :
535 : /* adjust file mode */
536 266 : if (state->opt.file_mode != ADVISE_DEFAULT) {
537 4 : state->file_mode = state->opt.file_mode;
538 : } else {
539 : /* default mode, if nothing is specified */
540 262 : state->file_mode = ADVISE_FLUSH;
541 : }
542 :
543 : /* store current command */
544 266 : state->command = command;
545 :
546 266 : log_tag("conf:file:%s\n", path);
547 :
548 266 : f = sopen_read(path);
549 266 : if (!f) {
550 : /* LCOV_EXCL_START */
551 : if (errno == ENOENT) {
552 : if (state->opt.auto_conf) {
553 : log_tag("conf:missing:\n");
554 : /* mark that we are without a configuration file */
555 : state->no_conf = 1;
556 : state->level = 0;
557 : return;
558 : }
559 :
560 : log_fatal("No configuration file found at '%s'\n", path);
561 : } else if (errno == EACCES) {
562 : log_fatal("You do not have rights to access the configuration file '%s'\n", path);
563 : } else {
564 : log_fatal("Error opening the configuration file '%s'. %s.\n", path, strerror(errno));
565 : }
566 : exit(EXIT_FAILURE);
567 : /* LCOV_EXCL_STOP */
568 : }
569 :
570 266 : line = 1;
571 : while (1) {
572 : char tag[PATH_MAX];
573 : char buffer[PATH_MAX];
574 : int ret;
575 : int c;
576 : unsigned level;
577 :
578 : /* skip initial spaces */
579 6245 : sgetspace(f);
580 :
581 : /* read the command */
582 6245 : ret = sgettok(f, tag, sizeof(tag));
583 6245 : if (ret < 0) {
584 : /* LCOV_EXCL_START */
585 : log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
586 : exit(EXIT_FAILURE);
587 : /* LCOV_EXCL_STOP */
588 : }
589 :
590 : /* skip spaces after the command */
591 6245 : sgetspace(f);
592 :
593 6245 : if (strcmp(tag, "blocksize") == 0
594 : /* block_size is the old format of the option */
595 5979 : || strcmp(tag, "block_size") == 0) {
596 :
597 266 : ret = sgetu32(f, &state->block_size);
598 266 : if (ret < 0) {
599 : /* LCOV_EXCL_START */
600 : log_fatal("Invalid 'blocksize' specification in '%s' at line %u\n", path, line);
601 : exit(EXIT_FAILURE);
602 : /* LCOV_EXCL_STOP */
603 : }
604 266 : if (state->block_size < 1) {
605 : /* LCOV_EXCL_START */
606 : log_fatal("Too small 'blocksize' specification in '%s' at line %u\n", path, line);
607 : exit(EXIT_FAILURE);
608 : /* LCOV_EXCL_STOP */
609 : }
610 266 : if (state->block_size > 16 * KIBI) {
611 : /* LCOV_EXCL_START */
612 : log_fatal("Too big 'blocksize' specification in '%s' at line %u\n", path, line);
613 : exit(EXIT_FAILURE);
614 : /* LCOV_EXCL_STOP */
615 : }
616 : /* check if it's a power of 2 */
617 266 : if ((state->block_size & (state->block_size - 1)) != 0) {
618 : /* LCOV_EXCL_START */
619 : log_fatal("Not power of 2 'blocksize' specification in '%s' at line %u\n", path, line);
620 : exit(EXIT_FAILURE);
621 : /* LCOV_EXCL_STOP */
622 : }
623 266 : state->block_size *= KIBI;
624 5979 : } else if (strcmp(tag, "hashsize") == 0
625 5979 : || strcmp(tag, "hash_size") == 0 /* v11.0 used incorretly this one, kept now for backward compatibility */
626 0 : ) {
627 : uint32_t hash_size;
628 :
629 0 : ret = sgetu32(f, &hash_size);
630 0 : if (ret < 0) {
631 : /* LCOV_EXCL_START */
632 : log_fatal("Invalid 'hashsize' specification in '%s' at line %u\n", path, line);
633 : exit(EXIT_FAILURE);
634 : /* LCOV_EXCL_STOP */
635 : }
636 0 : if (hash_size < 2) {
637 : /* LCOV_EXCL_START */
638 : log_fatal("Too small 'hashsize' specification in '%s' at line %u\n", path, line);
639 : exit(EXIT_FAILURE);
640 : /* LCOV_EXCL_STOP */
641 : }
642 0 : if (hash_size > HASH_MAX) {
643 : /* LCOV_EXCL_START */
644 : log_fatal("Too big 'hashsize' specification in '%s' at line %u\n", path, line);
645 : exit(EXIT_FAILURE);
646 : /* LCOV_EXCL_STOP */
647 : }
648 : /* check if it's a power of 2 */
649 0 : if ((hash_size & (hash_size - 1)) != 0) {
650 : /* LCOV_EXCL_START */
651 : log_fatal("Not power of 2 'hashsize' specification in '%s' at line %u\n", path, line);
652 : exit(EXIT_FAILURE);
653 : /* LCOV_EXCL_STOP */
654 : }
655 :
656 0 : BLOCK_HASH_SIZE = hash_size;
657 5979 : } else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
658 : char device[PATH_MAX];
659 : char* split_map[SPLIT_MAX+1];
660 : unsigned split_mac;
661 : char* slash;
662 : uint64_t dev;
663 : int skip_access;
664 :
665 1272 : if (state->parity[level].split_mac != 0) {
666 : /* LCOV_EXCL_START */
667 : log_fatal("Multiple '%s' specification in '%s' at line %u\n", tag, path, line);
668 : exit(EXIT_FAILURE);
669 : /* LCOV_EXCL_STOP */
670 : }
671 :
672 1272 : ret = sgetlasttok(f, buffer, sizeof(buffer));
673 1272 : if (ret < 0) {
674 : /* LCOV_EXCL_START */
675 : log_fatal("Invalid '%s' specification in '%s' at line %u\n", tag, path, line);
676 : exit(EXIT_FAILURE);
677 : /* LCOV_EXCL_STOP */
678 : }
679 :
680 1272 : if (!*buffer) {
681 : /* LCOV_EXCL_START */
682 : log_fatal("Empty '%s' specification in '%s' at line %u\n", tag, path, line);
683 : exit(EXIT_FAILURE);
684 : /* LCOV_EXCL_STOP */
685 : }
686 :
687 1272 : split_mac = strsplit(split_map, SPLIT_MAX+1, buffer, ",");
688 :
689 1272 : if (split_mac > SPLIT_MAX) {
690 : /* LCOV_EXCL_START */
691 : log_fatal("Too many files in '%s' specification in '%s' at line %u\n", tag, path, line);
692 : exit(EXIT_FAILURE);
693 : /* LCOV_EXCL_STOP */
694 : }
695 :
696 1272 : skip_access = 0;
697 1272 : state->parity[level].split_mac = split_mac;
698 6356 : for (s = 0; s < split_mac; ++s) {
699 5085 : pathimport(state->parity[level].split_map[s].path, sizeof(state->parity[level].split_map[s].path), split_map[s]);
700 :
701 5085 : if (!state->opt.skip_parity_access) {
702 : struct stat st;
703 :
704 : /* get the device of the directory containing the parity file */
705 4349 : pathimport(device, sizeof(device), split_map[s]);
706 4349 : slash = strrchr(device, '/');
707 4349 : if (slash)
708 4349 : *slash = 0;
709 : else
710 0 : pathcpy(device, sizeof(device), ".");
711 :
712 4349 : if (stat(device, &st) == 0) {
713 4344 : dev = st.st_dev;
714 : } else {
715 : /* if the disk can be skipped */
716 5 : if (state->opt.force_device) {
717 : /* use a fake device, and mark the disk to be skipped */
718 4 : dev = 0;
719 4 : skip_access = 1;
720 4 : log_fatal("DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
721 : } else {
722 : /* LCOV_EXCL_START */
723 : log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
724 :
725 : /* in "fix" we allow to continue anyway */
726 : if (strcmp(state->command, "fix") == 0) {
727 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
728 : }
729 : exit(EXIT_FAILURE);
730 : /* LCOV_EXCL_STOP */
731 : }
732 : }
733 : } else {
734 : /* use a fake device */
735 736 : dev = 0;
736 : }
737 :
738 5084 : state->parity[level].split_map[s].device = dev;
739 : }
740 :
741 : /* store the global parity skip_access */
742 1271 : state->parity[level].skip_access = skip_access;
743 :
744 : /* adjust the level */
745 1271 : if (state->level < level + 1)
746 1005 : state->level = level + 1;
747 4707 : } else if (strcmp(tag, "share") == 0) {
748 41 : if (*state->share) {
749 : /* LCOV_EXCL_START */
750 : log_fatal("Multiple 'share' specification in '%s' at line %u\n", path, line);
751 : exit(EXIT_FAILURE);
752 : /* LCOV_EXCL_STOP */
753 : }
754 :
755 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
756 41 : if (ret < 0) {
757 : /* LCOV_EXCL_START */
758 : log_fatal("Invalid 'share' specification in '%s' at line %u\n", path, line);
759 : exit(EXIT_FAILURE);
760 : /* LCOV_EXCL_STOP */
761 : }
762 :
763 41 : if (!*buffer) {
764 : /* LCOV_EXCL_START */
765 : log_fatal("Empty 'share' specification in '%s' at line %u\n", path, line);
766 : exit(EXIT_FAILURE);
767 : /* LCOV_EXCL_STOP */
768 : }
769 :
770 41 : pathimport(state->share, sizeof(state->share), buffer);
771 4666 : } else if (strcmp(tag, "pool") == 0) {
772 : struct stat st;
773 :
774 41 : if (*state->pool) {
775 : /* LCOV_EXCL_START */
776 : log_fatal("Multiple 'pool' specification in '%s' at line %u\n", path, line);
777 : exit(EXIT_FAILURE);
778 : /* LCOV_EXCL_STOP */
779 : }
780 :
781 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
782 41 : if (ret < 0) {
783 : /* LCOV_EXCL_START */
784 : log_fatal("Invalid 'pool' specification in '%s' at line %u\n", path, line);
785 : exit(EXIT_FAILURE);
786 : /* LCOV_EXCL_STOP */
787 : }
788 :
789 41 : if (!*buffer) {
790 : /* LCOV_EXCL_START */
791 : log_fatal("Empty 'pool' specification in '%s' at line %u\n", path, line);
792 : exit(EXIT_FAILURE);
793 : /* LCOV_EXCL_STOP */
794 : }
795 :
796 41 : pathimport(state->pool, sizeof(state->pool), buffer);
797 :
798 : /* get the device of the directory containing the pool tree */
799 41 : if (stat(buffer, &st) != 0) {
800 : /* LCOV_EXCL_START */
801 : log_fatal("Error accessing 'pool' dir '%s' specification in '%s' at line %u\n", buffer, path, line);
802 : exit(EXIT_FAILURE);
803 : /* LCOV_EXCL_STOP */
804 : }
805 :
806 41 : state->pool_device = st.st_dev;
807 4625 : } else if (strcmp(tag, "content") == 0) {
808 : struct snapraid_content* content;
809 : char device[PATH_MAX];
810 : char* slash;
811 : struct stat st;
812 : uint64_t dev;
813 :
814 1531 : ret = sgetlasttok(f, buffer, sizeof(buffer));
815 1531 : if (ret < 0) {
816 : /* LCOV_EXCL_START */
817 : log_fatal("Invalid 'content' specification in '%s' at line %u\n", path, line);
818 : exit(EXIT_FAILURE);
819 : /* LCOV_EXCL_STOP */
820 : }
821 :
822 1531 : if (pathcmp(buffer, "/dev/null") == 0 || pathcmp(buffer, "NUL") == 0) {
823 : /* LCOV_EXCL_START */
824 : log_fatal("You cannot use the null device as 'content' specification in '%s' at line %u\n", path, line);
825 : exit(EXIT_FAILURE);
826 : /* LCOV_EXCL_STOP */
827 : }
828 :
829 1531 : if (!*buffer) {
830 : /* LCOV_EXCL_START */
831 : log_fatal("Empty 'content' specification in '%s' at line %u\n", path, line);
832 : exit(EXIT_FAILURE);
833 : /* LCOV_EXCL_STOP */
834 : }
835 :
836 : /* check if the content file is already specified */
837 5724 : for (i = state->contentlist; i != 0; i = i->next) {
838 4193 : content = i->data;
839 4193 : if (pathcmp(content->content, buffer) == 0)
840 0 : break;
841 : }
842 1531 : if (i) {
843 : /* LCOV_EXCL_START */
844 : log_fatal("Duplicate 'content' specification in '%s' at line %u\n", path, line);
845 : exit(EXIT_FAILURE);
846 : /* LCOV_EXCL_STOP */
847 : }
848 :
849 : /* get the device of the directory containing the content file */
850 1531 : pathimport(device, sizeof(device), buffer);
851 1531 : slash = strrchr(device, '/');
852 1531 : if (slash)
853 1531 : *slash = 0;
854 : else
855 0 : pathcpy(device, sizeof(device), ".");
856 1531 : if (stat(device, &st) == 0) {
857 1530 : dev = st.st_dev;
858 : } else {
859 1 : if (state->opt.skip_content_access) {
860 : /* use a fake device */
861 1 : dev = 0;
862 1 : log_fatal("WARNING! Skipping inaccessible content file '%s'...\n", buffer);
863 : } else {
864 : /* LCOV_EXCL_START */
865 : log_fatal("Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
866 : exit(EXIT_FAILURE);
867 : /* LCOV_EXCL_STOP */
868 : }
869 : }
870 :
871 : /* set the lock file at the first accessible content file */
872 1531 : if (state->lockfile[0] == 0 && dev != 0) {
873 265 : pathcpy(state->lockfile, sizeof(state->lockfile), buffer);
874 265 : pathcat(state->lockfile, sizeof(state->lockfile), ".lock");
875 : }
876 :
877 1531 : content = content_alloc(buffer, dev);
878 :
879 1531 : tommy_list_insert_tail(&state->contentlist, &content->node, content);
880 4679 : } else if (strcmp(tag, "data") == 0 || strcmp(tag, "disk") == 0) {
881 : /* "disk" is the deprecated name up to SnapRAID 9.x */
882 : char dir[PATH_MAX];
883 : char device[PATH_MAX];
884 : char uuid[UUID_MAX];
885 : struct snapraid_disk* disk;
886 : uint64_t dev;
887 : int skip_access;
888 :
889 1585 : ret = sgettok(f, buffer, sizeof(buffer));
890 1585 : if (ret < 0) {
891 : /* LCOV_EXCL_START */
892 : log_fatal("Invalid 'data' name specification in '%s' at line %u\n", path, line);
893 : exit(EXIT_FAILURE);
894 : /* LCOV_EXCL_STOP */
895 : }
896 :
897 1585 : if (!*buffer) {
898 : /* LCOV_EXCL_START */
899 : log_fatal("Empty 'data' name specification in '%s' at line %u\n", path, line);
900 : exit(EXIT_FAILURE);
901 : /* LCOV_EXCL_STOP */
902 : }
903 :
904 1585 : sgetspace(f);
905 :
906 1585 : ret = sgetlasttok(f, dir, sizeof(dir));
907 1585 : if (ret < 0) {
908 : /* LCOV_EXCL_START */
909 : log_fatal("Invalid 'data' dir specification in '%s' at line %u\n", path, line);
910 : exit(EXIT_FAILURE);
911 : /* LCOV_EXCL_STOP */
912 : }
913 :
914 1585 : if (!*dir) {
915 : /* LCOV_EXCL_START */
916 : log_fatal("Empty 'data' dir specification in '%s' at line %u\n", path, line);
917 : exit(EXIT_FAILURE);
918 : /* LCOV_EXCL_STOP */
919 : }
920 :
921 : /* get the device of the dir */
922 1585 : pathimport(device, sizeof(device), dir);
923 :
924 : /* check if the disk name already exists */
925 5535 : for (i = state->disklist; i != 0; i = i->next) {
926 3950 : disk = i->data;
927 3950 : if (strcmp(disk->name, buffer) == 0)
928 0 : break;
929 : }
930 1585 : if (i) {
931 : /* LCOV_EXCL_START */
932 : log_fatal("Duplicate 'data' name '%s' at line %u\n", buffer, line);
933 : exit(EXIT_FAILURE);
934 : /* LCOV_EXCL_STOP */
935 : }
936 :
937 : /* if the disk has to be present */
938 1585 : skip_access = 0;
939 1585 : if (!state->opt.skip_disk_access) {
940 : struct stat st;
941 :
942 1423 : if (stat(device, &st) == 0) {
943 1422 : dev = st.st_dev;
944 :
945 : /* read the uuid, if unsupported use an empty one */
946 1422 : if (devuuid(dev, uuid, sizeof(uuid)) != 0) {
947 0 : *uuid = 0;
948 : }
949 :
950 : /* fake a different UUID when testing */
951 1422 : if (state->opt.fake_uuid) {
952 2 : snprintf(uuid, sizeof(uuid), "fake-uuid-%d", state->opt.fake_uuid);
953 2 : --state->opt.fake_uuid;
954 : }
955 : } else {
956 : /* if the disk can be skipped */
957 1 : if (state->opt.force_device) {
958 : /* use a fake device, and mark the disk to be skipped */
959 1 : dev = 0;
960 1 : *uuid = 0;
961 1 : skip_access = 1;
962 1 : log_fatal("DANGER! Skipping inaccessible data disk '%s'...\n", buffer);
963 : } else {
964 : /* LCOV_EXCL_START */
965 : log_fatal("Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
966 :
967 : /* in "fix" we allow to continue anyway */
968 : if (strcmp(state->command, "fix") == 0) {
969 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
970 : }
971 : exit(EXIT_FAILURE);
972 : /* LCOV_EXCL_STOP */
973 : }
974 : }
975 : } else {
976 : /* use a fake device */
977 162 : dev = 0;
978 162 : *uuid = 0;
979 : }
980 :
981 1585 : disk = disk_alloc(buffer, dir, dev, uuid, skip_access);
982 :
983 1585 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
984 1509 : } else if (strcmp(tag, "smartctl") == 0) {
985 : char custom[PATH_MAX];
986 :
987 366 : ret = sgettok(f, buffer, sizeof(buffer));
988 366 : if (ret < 0) {
989 : /* LCOV_EXCL_START */
990 : log_fatal("Invalid 'smartctl' name specification in '%s' at line %u\n", path, line);
991 : exit(EXIT_FAILURE);
992 : /* LCOV_EXCL_STOP */
993 : }
994 :
995 366 : if (!*buffer) {
996 : /* LCOV_EXCL_START */
997 : log_fatal("Empty 'smartctl' name specification in '%s' at line %u\n", path, line);
998 : exit(EXIT_FAILURE);
999 : /* LCOV_EXCL_STOP */
1000 : }
1001 :
1002 366 : sgetspace(f);
1003 :
1004 366 : ret = sgetlasttok(f, custom, sizeof(custom));
1005 366 : if (ret < 0) {
1006 : /* LCOV_EXCL_START */
1007 : log_fatal("Invalid 'smartctl' option specification in '%s' at line %u\n", path, line);
1008 : exit(EXIT_FAILURE);
1009 : /* LCOV_EXCL_STOP */
1010 : }
1011 :
1012 366 : if (!*custom) {
1013 : /* LCOV_EXCL_START */
1014 : log_fatal("Empty 'smartctl' option specification in '%s' at line %u\n", path, line);
1015 : exit(EXIT_FAILURE);
1016 : /* LCOV_EXCL_STOP */
1017 : }
1018 :
1019 366 : if (validate_smartctl(custom) != 0) {
1020 : /* LCOV_EXCL_START */
1021 : log_fatal("Invalid 'smartctl' option specification in '%s' at line %u\n", path, line);
1022 : exit(EXIT_FAILURE);
1023 : /* LCOV_EXCL_STOP */
1024 : }
1025 :
1026 : /* search for parity */
1027 366 : if (lev_config_scan(buffer, &level, 0) == 0) {
1028 183 : if (state->parity[level].smartctl[0] != 0) {
1029 : /* LCOV_EXCL_START */
1030 : log_fatal("Duplicate parity smartctl '%s' at line %u\n", buffer, line);
1031 : exit(EXIT_FAILURE);
1032 : /* LCOV_EXCL_STOP */
1033 : }
1034 :
1035 183 : pathcpy(state->parity[level].smartctl, sizeof(state->parity[level].smartctl), custom);
1036 : } else {
1037 : struct snapraid_disk* disk;
1038 :
1039 : /* search the disk */
1040 183 : disk = 0;
1041 183 : for (i = state->disklist; i != 0; i = i->next) {
1042 183 : disk = i->data;
1043 183 : if (strcmp(disk->name, buffer) == 0)
1044 183 : break;
1045 : }
1046 183 : if (!disk) {
1047 : /* LCOV_EXCL_START */
1048 : log_fatal("Missing disk smartctl '%s' at line %u\n", buffer, line);
1049 : exit(EXIT_FAILURE);
1050 : /* LCOV_EXCL_STOP */
1051 : }
1052 183 : if (disk->smartctl[0] != 0) {
1053 : /* LCOV_EXCL_START */
1054 : log_fatal("Duplicate disk name '%s' at line %u\n", buffer, line);
1055 : exit(EXIT_FAILURE);
1056 : /* LCOV_EXCL_STOP */
1057 : }
1058 :
1059 183 : pathcpy(disk->smartctl, sizeof(disk->smartctl), custom);
1060 : }
1061 1143 : } else if (strcmp(tag, "nohidden") == 0) {
1062 0 : state->filter_hidden = 1;
1063 1143 : } else if (strcmp(tag, "exclude") == 0) {
1064 : struct snapraid_filter* filter;
1065 :
1066 265 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1067 265 : if (ret < 0) {
1068 : /* LCOV_EXCL_START */
1069 : log_fatal("Invalid 'exclude' specification in '%s' at line %u\n", path, line);
1070 : exit(EXIT_FAILURE);
1071 : /* LCOV_EXCL_STOP */
1072 : }
1073 :
1074 265 : if (!*buffer) {
1075 : /* LCOV_EXCL_START */
1076 : log_fatal("Empty 'exclude' specification in '%s' at line %u\n", path, line);
1077 : exit(EXIT_FAILURE);
1078 : /* LCOV_EXCL_STOP */
1079 : }
1080 :
1081 265 : filter = filter_alloc_file(-1, buffer);
1082 265 : if (!filter) {
1083 : /* LCOV_EXCL_START */
1084 : log_fatal("Invalid 'exclude' specification '%s' in '%s' at line %u\n", buffer, path, line);
1085 : log_fatal("Filters using relative paths are not supported. Ensure to add an initial slash\n");
1086 : exit(EXIT_FAILURE);
1087 : /* LCOV_EXCL_STOP */
1088 : }
1089 265 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1090 878 : } else if (strcmp(tag, "include") == 0) {
1091 : struct snapraid_filter* filter;
1092 :
1093 265 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1094 265 : if (ret < 0) {
1095 : /* LCOV_EXCL_START */
1096 : log_fatal("Invalid 'include' specification in '%s' at line %u\n", path, line);
1097 : exit(EXIT_FAILURE);
1098 : /* LCOV_EXCL_STOP */
1099 : }
1100 :
1101 265 : if (!*buffer) {
1102 : /* LCOV_EXCL_START */
1103 : log_fatal("Empty 'include' specification in '%s' at line %u\n", path, line);
1104 : exit(EXIT_FAILURE);
1105 : /* LCOV_EXCL_STOP */
1106 : }
1107 :
1108 265 : filter = filter_alloc_file(1, buffer);
1109 265 : if (!filter) {
1110 : /* LCOV_EXCL_START */
1111 : log_fatal("Invalid 'include' specification '%s' in '%s' at line %u\n", buffer, path, line);
1112 : log_fatal("Filters using relative paths are not supported. Ensure to add an initial slash\n");
1113 : exit(EXIT_FAILURE);
1114 : /* LCOV_EXCL_STOP */
1115 : }
1116 265 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1117 613 : } else if (strcmp(tag, "autosave") == 0) {
1118 : char* e;
1119 :
1120 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1121 41 : if (ret < 0) {
1122 : /* LCOV_EXCL_START */
1123 : log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1124 : exit(EXIT_FAILURE);
1125 : /* LCOV_EXCL_STOP */
1126 : }
1127 :
1128 41 : if (!*buffer) {
1129 : /* LCOV_EXCL_START */
1130 : log_fatal("Empty 'autosave' specification in '%s' at line %u\n", path, line);
1131 : exit(EXIT_FAILURE);
1132 : /* LCOV_EXCL_STOP */
1133 : }
1134 :
1135 41 : state->autosave = strtoul(buffer, &e, 0);
1136 :
1137 41 : if (!e || *e) {
1138 : /* LCOV_EXCL_START */
1139 : log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1140 : exit(EXIT_FAILURE);
1141 : /* LCOV_EXCL_STOP */
1142 : }
1143 :
1144 : /* convert to GB */
1145 41 : state->autosave *= GIGA;
1146 572 : } else if (tag[0] == 0) {
1147 : /* allow empty lines */
1148 41 : } else if (tag[0] == '#') {
1149 41 : ret = sgetline(f, buffer, sizeof(buffer));
1150 41 : if (ret < 0) {
1151 : /* LCOV_EXCL_START */
1152 : log_fatal("Invalid comment in '%s' at line %u\n", path, line);
1153 : exit(EXIT_FAILURE);
1154 : /* LCOV_EXCL_STOP */
1155 : }
1156 : } else {
1157 : /* LCOV_EXCL_START */
1158 : log_fatal("Invalid command '%s' in '%s' at line %u\n", tag, path, line);
1159 : exit(EXIT_FAILURE);
1160 : /* LCOV_EXCL_STOP */
1161 : }
1162 :
1163 : /* skip final spaces */
1164 6244 : sgetspace(f);
1165 :
1166 : /* next line */
1167 6244 : c = sgeteol(f);
1168 6244 : if (c == EOF) {
1169 265 : break;
1170 : }
1171 5979 : if (c != '\n') {
1172 : /* LCOV_EXCL_START */
1173 : log_fatal("Extra data in '%s' at line %u\n", path, line);
1174 : exit(EXIT_FAILURE);
1175 : /* LCOV_EXCL_STOP */
1176 : }
1177 5979 : ++line;
1178 5979 : }
1179 :
1180 265 : if (serror(f)) {
1181 : /* LCOV_EXCL_START */
1182 : log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
1183 : exit(EXIT_FAILURE);
1184 : /* LCOV_EXCL_STOP */
1185 : }
1186 :
1187 265 : sclose(f);
1188 :
1189 265 : state_config_check(state, path, filterlist_disk);
1190 :
1191 : /* select the default hash */
1192 265 : if (state->opt.force_murmur3) {
1193 1 : state->besthash = HASH_MURMUR3;
1194 264 : } else if (state->opt.force_spooky2) {
1195 1 : state->besthash = HASH_SPOOKY2;
1196 : } else {
1197 : #ifdef CONFIG_X86
1198 263 : if (sizeof(void*) == 4 && !raid_cpu_has_slowmult())
1199 263 : state->besthash = HASH_MURMUR3;
1200 : else
1201 0 : state->besthash = HASH_SPOOKY2;
1202 : #else
1203 : if (sizeof(void*) == 4)
1204 : state->besthash = HASH_MURMUR3;
1205 : else
1206 : state->besthash = HASH_SPOOKY2;
1207 : #endif
1208 : }
1209 :
1210 : /* by default use the best hash */
1211 265 : state->hash = state->besthash;
1212 :
1213 : /* by default use a random hash seed */
1214 265 : if (randomize(state->hashseed, HASH_MAX) != 0) {
1215 : /* LCOV_EXCL_START */
1216 : log_fatal("Failed to get random values.\n");
1217 : exit(EXIT_FAILURE);
1218 : /* LCOV_EXCL_STOP */
1219 : }
1220 :
1221 : /* no previous hash by default */
1222 265 : state->prevhash = HASH_UNDEFINED;
1223 :
1224 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
1225 :
1226 265 : log_tag("blocksize:%u\n", state->block_size);
1227 1850 : for (i = state->disklist; i != 0; i = i->next) {
1228 1585 : struct snapraid_disk* disk = i->data;
1229 1585 : log_tag("data:%s:%s\n", disk->name, disk->dir);
1230 : }
1231 :
1232 265 : log_tag("mode:%s\n", lev_raid_name(state->raid_mode, state->level));
1233 1531 : for (l = 0; l < state->level; ++l)
1234 6330 : for (s = 0; s < state->parity[l].split_mac; ++s)
1235 5064 : log_tag("%s:%u:%s\n", lev_config_name(l), s, state->parity[l].split_map[s].path);
1236 265 : if (state->pool[0] != 0)
1237 41 : log_tag("pool:%s\n", state->pool);
1238 265 : if (state->share[0] != 0)
1239 41 : log_tag("share:%s\n", state->share);
1240 265 : if (state->autosave != 0)
1241 41 : log_tag("autosave:%" PRIu64 "\n", state->autosave);
1242 795 : for (i = tommy_list_head(&state->filterlist); i != 0; i = i->next) {
1243 : char out[PATH_MAX];
1244 530 : struct snapraid_filter* filter = i->data;
1245 530 : log_tag("filter:%s\n", filter_type(filter, out, sizeof(out)));
1246 : }
1247 265 : if (state->filter_hidden)
1248 0 : log_tag("filter:nohidden:\n");
1249 265 : log_flush();
1250 : }
1251 :
1252 : /**
1253 : * Find a disk by name.
1254 : */
1255 7047 : static struct snapraid_disk* find_disk_by_name(struct snapraid_state* state, const char* name)
1256 : {
1257 : tommy_node* i;
1258 :
1259 24588 : for (i = state->disklist; i != 0; i = i->next) {
1260 24580 : struct snapraid_disk* disk = i->data;
1261 24580 : if (strcmp(disk->name, name) == 0)
1262 7039 : return disk;
1263 : }
1264 :
1265 8 : if (state->no_conf) {
1266 : /* without a configuration file, add disks automatically */
1267 : struct snapraid_disk* disk;
1268 :
1269 6 : disk = disk_alloc(name, "DUMMY/", -1, "", 0);
1270 :
1271 6 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
1272 :
1273 6 : return disk;
1274 : }
1275 :
1276 2 : return 0;
1277 : }
1278 :
1279 : /**
1280 : * Find a disk by UUID.
1281 : */
1282 2 : static struct snapraid_disk* find_disk_by_uuid(struct snapraid_state* state, const char* uuid)
1283 : {
1284 : tommy_node* i;
1285 2 : struct snapraid_disk* found = 0;
1286 :
1287 : /* special test case to find the first matching UUID */
1288 : /* when testing UUID are all equal or not supported */
1289 : /* and we should handle this case specifically */
1290 2 : if (state->opt.match_first_uuid)
1291 2 : return state->disklist->data;
1292 :
1293 : /* LCOV_EXCL_START */
1294 : /* never find an empty uuid */
1295 : if (!*uuid)
1296 : return 0;
1297 :
1298 : for (i = state->disklist; i != 0; i = i->next) {
1299 : struct snapraid_disk* disk = i->data;
1300 : if (strcmp(disk->uuid, uuid) == 0) {
1301 : /* never match duplicate UUID */
1302 : if (found)
1303 : return 0;
1304 : found = disk;
1305 : }
1306 : }
1307 :
1308 : return found;
1309 : /* LCOV_EXCL_STOP */
1310 : }
1311 :
1312 : /**
1313 : * Update the disk mapping if required.
1314 : */
1315 260 : static void state_map(struct snapraid_state* state)
1316 : {
1317 : unsigned hole;
1318 : tommy_node* i;
1319 : unsigned uuid_mismatch;
1320 : unsigned diskcount;
1321 : unsigned l, s;
1322 :
1323 : /* remove all the mapping without a disk */
1324 : /* this happens when a disk is removed from the configuration file */
1325 : /* From SnapRAID 4.0 mappings are automatically removed if a disk is not used */
1326 : /* when saving the content file, but we keep this code to import older content files. */
1327 1917 : for (i = state->maplist; i != 0; ) {
1328 1397 : struct snapraid_map* map = i->data;
1329 : struct snapraid_disk* disk;
1330 :
1331 1397 : disk = find_disk_by_name(state, map->name);
1332 :
1333 : /* go to the next mapping before removing */
1334 1397 : i = i->next;
1335 :
1336 1397 : if (disk == 0) {
1337 : /* disk not found, remove the mapping */
1338 0 : tommy_list_remove_existing(&state->maplist, &map->node);
1339 0 : map_free(map);
1340 : }
1341 : }
1342 :
1343 : /* maps each unmapped disk present in the configuration file in the first available hole */
1344 : /* this happens when you add disks for the first time in the configuration file */
1345 260 : hole = 0; /* first position to try */
1346 1815 : for (i = state->disklist; i != 0; i = i->next) {
1347 1555 : struct snapraid_disk* disk = i->data;
1348 : struct snapraid_map* map;
1349 : tommy_node* j;
1350 :
1351 : /* check if the disk is already mapped */
1352 5430 : for (j = state->maplist; j != 0; j = j->next) {
1353 5272 : map = j->data;
1354 5272 : if (strcmp(disk->name, map->name) == 0) {
1355 : /* mapping found */
1356 1397 : break;
1357 : }
1358 : }
1359 1555 : if (j != 0) {
1360 : /* mapping is present, then copy the free blocks into to disk */
1361 1397 : disk->total_blocks = map->total_blocks;
1362 1397 : disk->free_blocks = map->free_blocks;
1363 1397 : continue;
1364 : }
1365 :
1366 : /* mapping not found, search for an hole */
1367 : while (1) {
1368 1100 : for (j = state->maplist; j != 0; j = j->next) {
1369 942 : map = j->data;
1370 942 : if (map->position == hole) {
1371 : /* position already used */
1372 158 : break;
1373 : }
1374 : }
1375 316 : if (j == 0) {
1376 : /* hole found */
1377 158 : break;
1378 : }
1379 :
1380 : /* try with the next one */
1381 158 : ++hole;
1382 158 : }
1383 :
1384 : /* insert the new mapping */
1385 158 : map = map_alloc(disk->name, hole, 0, 0, "");
1386 :
1387 158 : tommy_list_insert_tail(&state->maplist, &map->node, map);
1388 : }
1389 :
1390 : /* without configuration don't check for number of data disks or uuid changes */
1391 260 : if (state->no_conf)
1392 261 : return;
1393 :
1394 : /* counter for the number of UUID mismatches */
1395 259 : uuid_mismatch = 0;
1396 :
1397 : /* check if mapping match the disk uuid */
1398 259 : if (!state->opt.skip_disk_access) {
1399 1626 : for (i = state->maplist; i != 0; i = i->next) {
1400 1393 : struct snapraid_map* map = i->data;
1401 : struct snapraid_disk* disk;
1402 :
1403 1393 : disk = find_disk_by_name(state, map->name);
1404 1393 : if (disk == 0) {
1405 : /* LCOV_EXCL_START */
1406 : log_fatal("Internal inconsistency for mapping '%s'\n", map->name);
1407 : os_abort();
1408 : /* LCOV_EXCL_STOP */
1409 : }
1410 :
1411 1393 : if (disk->has_unsupported_uuid) {
1412 : /* if uuid is not available, skip this one */
1413 1 : continue;
1414 : }
1415 :
1416 : /* if the uuid is changed */
1417 1392 : if (strcmp(disk->uuid, map->uuid) != 0) {
1418 : /* mark the disk as with an UUID change */
1419 113 : disk->has_different_uuid = 1;
1420 :
1421 : /* if the previous uuid is available */
1422 113 : if (map->uuid[0] != 0) {
1423 : /* count the number of uuid change */
1424 4 : ++uuid_mismatch;
1425 4 : log_fatal("UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, disk->uuid);
1426 : } else {
1427 : /* no message here, because having a disk without */
1428 : /* UUID is the normal state of an empty disk */
1429 109 : disk->had_empty_uuid = 1;
1430 : }
1431 :
1432 : /* update the uuid in the mapping, */
1433 113 : pathcpy(map->uuid, sizeof(map->uuid), disk->uuid);
1434 :
1435 : /* write the new state with the new uuid */
1436 113 : state->need_write = 1;
1437 : }
1438 : }
1439 : }
1440 :
1441 : /* check the parity uuid */
1442 259 : if (!state->opt.skip_parity_access) {
1443 1273 : for (l = 0; l < state->level; ++l) {
1444 5260 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1445 : char uuid[UUID_MAX];
1446 : int ret;
1447 :
1448 4208 : ret = devuuid(state->parity[l].split_map[s].device, uuid, sizeof(uuid));
1449 4208 : if (ret != 0) {
1450 : /* uuid not available, just ignore */
1451 4 : continue;
1452 : }
1453 :
1454 : /* if the uuid is changed */
1455 4204 : if (strcmp(uuid, state->parity[l].split_map[s].uuid) != 0) {
1456 : /* if the previous uuid is available */
1457 184 : if (state->parity[l].split_map[s].uuid[0] != 0) {
1458 : /* count the number of uuid change */
1459 0 : ++uuid_mismatch;
1460 0 : log_fatal("UUID change for parity '%s[%u]' from '%s' to '%s'\n", lev_config_name(l), s, state->parity[l].split_map[s].uuid, uuid);
1461 : }
1462 :
1463 : /* update the uuid */
1464 184 : pathcpy(state->parity[l].split_map[s].uuid, sizeof(state->parity[l].split_map[s].uuid), uuid);
1465 :
1466 : /* write the new state with the new uuid */
1467 184 : state->need_write = 1;
1468 : }
1469 : }
1470 : }
1471 : }
1472 :
1473 259 : if (!state->opt.force_uuid && uuid_mismatch > state->level) {
1474 : /* LCOV_EXCL_START */
1475 : log_fatal("Too many disks have UUID changed from the latest 'sync'.\n");
1476 : log_fatal("If this happens because you really replaced them,\n");
1477 : log_fatal("you can '%s' anyway, using 'snapraid --force-uuid %s'.\n", state->command, state->command);
1478 : log_fatal("Instead, it's possible that you messed up the disk mount points,\n");
1479 : log_fatal("and you have to restore the mount points at the state of the latest sync.\n");
1480 : exit(EXIT_FAILURE);
1481 : /* LCOV_EXCL_STOP */
1482 : }
1483 :
1484 : /* count the number of data disks, including holes left after removing some */
1485 259 : diskcount = 0;
1486 1808 : for (i = state->maplist; i != 0; i = i->next) {
1487 1549 : struct snapraid_map* map = i->data;
1488 :
1489 1549 : if (map->position + 1 > diskcount)
1490 1367 : diskcount = map->position + 1;
1491 : }
1492 :
1493 : /* ensure to don't go over the limit of the RAID engine */
1494 259 : if (diskcount > RAID_DATA_MAX) {
1495 : /* LCOV_EXCL_START */
1496 : log_fatal("Too many data disks. No more than %u.\n", RAID_DATA_MAX);
1497 : exit(EXIT_FAILURE);
1498 : /* LCOV_EXCL_STOP */
1499 : }
1500 :
1501 : /* now count the real number of data disks, excluding holes left after removing some */
1502 259 : diskcount = tommy_list_count(&state->maplist);
1503 :
1504 : /* recommend number of parities */
1505 259 : if (!state->opt.no_warnings) {
1506 : /* intentionally use log_fatal() instead of log_error() to give more visibility at the warning */
1507 1 : if (diskcount >= 36 && state->level < 6) {
1508 0 : log_fatal("WARNING! With %u disks it's recommended to use six parity levels.\n", diskcount);
1509 1 : } else if (diskcount >= 29 && state->level < 5) {
1510 0 : log_fatal("WARNING! With %u disks it's recommended to use five parity levels.\n", diskcount);
1511 1 : } else if (diskcount >= 22 && state->level < 4) {
1512 0 : log_fatal("WARNING! With %u disks it's recommended to use four parity levels.\n", diskcount);
1513 1 : } else if (diskcount >= 15 && state->level < 3) {
1514 0 : log_fatal("WARNING! With %u disks it's recommended to use three parity levels.\n", diskcount);
1515 1 : } else if (diskcount >= 5 && state->level < 2) {
1516 0 : log_fatal("WARNING! With %u disks it's recommended to use two parity levels.\n", diskcount);
1517 : }
1518 : }
1519 : }
1520 :
1521 173 : void state_refresh(struct snapraid_state* state)
1522 : {
1523 : tommy_node* i;
1524 : unsigned l, s;
1525 :
1526 : /* for all disks */
1527 1209 : for (i = state->maplist; i != 0; i = i->next) {
1528 1036 : struct snapraid_map* map = i->data;
1529 : struct snapraid_disk* disk;
1530 : uint64_t total_space;
1531 : uint64_t free_space;
1532 : int ret;
1533 :
1534 1036 : disk = find_disk_by_name(state, map->name);
1535 1036 : if (disk == 0) {
1536 : /* LCOV_EXCL_START */
1537 : log_fatal("Internal inconsistency for mapping '%s'\n", map->name);
1538 : os_abort();
1539 : /* LCOV_EXCL_STOP */
1540 : }
1541 :
1542 1036 : ret = fsinfo(disk->dir, 0, &total_space, &free_space);
1543 1036 : if (ret != 0) {
1544 : /* LCOV_EXCL_START */
1545 : log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
1546 : exit(EXIT_FAILURE);
1547 : /* LCOV_EXCL_STOP */
1548 : }
1549 :
1550 : /* set the new free blocks */
1551 1036 : map->total_blocks = total_space / state->block_size;
1552 1036 : map->free_blocks = free_space / state->block_size;
1553 :
1554 : /* also update the disk info */
1555 1036 : disk->total_blocks = map->total_blocks;
1556 1036 : disk->free_blocks = map->free_blocks;
1557 : }
1558 :
1559 : /* for all parities */
1560 1141 : for (l = 0; l < state->level; ++l) {
1561 : /* set the new free blocks */
1562 968 : state->parity[l].total_blocks = 0;
1563 968 : state->parity[l].free_blocks = 0;
1564 :
1565 4840 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1566 : uint64_t total_space;
1567 : uint64_t free_space;
1568 : int ret;
1569 :
1570 3872 : ret = fsinfo(state->parity[l].split_map[s].path, 0, &total_space, &free_space);
1571 3872 : if (ret != 0) {
1572 : /* LCOV_EXCL_START */
1573 : log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].split_map[s].path, strerror(errno));
1574 : exit(EXIT_FAILURE);
1575 : /* LCOV_EXCL_STOP */
1576 : }
1577 :
1578 : /* add the new free blocks */
1579 3872 : state->parity[l].total_blocks += total_space / state->block_size;
1580 3872 : state->parity[l].free_blocks += free_space / state->block_size;
1581 : }
1582 : }
1583 :
1584 : /* note what we don't set need_write = 1, because we don't want */
1585 : /* to update the content file only for the free space info. */
1586 173 : }
1587 :
1588 : /**
1589 : * Check the content.
1590 : */
1591 253 : static void state_content_check(struct snapraid_state* state, const char* path)
1592 : {
1593 : tommy_node* i;
1594 :
1595 : /* check that any map has different name and position */
1596 1766 : for (i = state->maplist; i != 0; i = i->next) {
1597 1513 : struct snapraid_map* map = i->data;
1598 : tommy_node* j;
1599 5283 : for (j = i->next; j != 0; j = j->next) {
1600 3770 : struct snapraid_map* other = j->data;
1601 3770 : if (strcmp(map->name, other->name) == 0) {
1602 : /* LCOV_EXCL_START */
1603 : log_fatal("Colliding 'map' disk specification in '%s'\n", path);
1604 : exit(EXIT_FAILURE);
1605 : /* LCOV_EXCL_STOP */
1606 : }
1607 3770 : if (map->position == other->position) {
1608 : /* LCOV_EXCL_START */
1609 : log_fatal("Colliding 'map' index specification in '%s'\n", path);
1610 : exit(EXIT_FAILURE);
1611 : /* LCOV_EXCL_STOP */
1612 : }
1613 : }
1614 : }
1615 253 : }
1616 :
1617 : /**
1618 : * Check if the position is REQUIRED, or we can completely clear it from the state.
1619 : *
1620 : * Note that position with only DELETED blocks are discarged.
1621 : */
1622 756099 : static int fs_position_is_required(struct snapraid_state* state, block_off_t pos)
1623 : {
1624 : tommy_node* i;
1625 :
1626 : /* check for each disk */
1627 802725 : for (i = state->disklist; i != 0; i = i->next) {
1628 802725 : struct snapraid_disk* disk = i->data;
1629 802725 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1630 :
1631 : /* if we have at least one file, the position is needed */
1632 802725 : if (block_has_file(block))
1633 756099 : return 1;
1634 : }
1635 :
1636 0 : return 0;
1637 : }
1638 :
1639 : /**
1640 : * Check if the info block is REQUIREQ.
1641 : *
1642 : * This is used to ensure that we keep the last check used for scrubbing.
1643 : * and that we add it when importing old context files.
1644 : *
1645 : * Note that you can have position without info blocks, for example
1646 : * if all the blocks are not synced.
1647 : *
1648 : * Note also that not requiring an info block, doesn't mean that if present it
1649 : * can be discarded.
1650 : */
1651 1547923 : static int fs_info_is_required(struct snapraid_state* state, block_off_t pos)
1652 : {
1653 : tommy_node* i;
1654 :
1655 : /* check for each disk */
1656 2048488 : for (i = state->disklist; i != 0; i = i->next) {
1657 1989723 : struct snapraid_disk* disk = i->data;
1658 1989723 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1659 :
1660 : /* if we have at least one synced file, the info is required */
1661 1989723 : if (block_state_get(block) == BLOCK_STATE_BLK)
1662 1489158 : return 1;
1663 : }
1664 :
1665 58765 : return 0;
1666 : }
1667 :
1668 0 : static void fs_position_clear_deleted(struct snapraid_state* state, block_off_t pos)
1669 : {
1670 : tommy_node* i;
1671 :
1672 : /* check for each disk if block is really used */
1673 0 : for (i = state->disklist; i != 0; i = i->next) {
1674 0 : struct snapraid_disk* disk = i->data;
1675 0 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1676 :
1677 : /* if the block is deleted */
1678 0 : if (block_state_get(block) == BLOCK_STATE_DELETED) {
1679 : /* set it to empty */
1680 0 : fs_deallocate(disk, pos);
1681 : }
1682 : }
1683 0 : }
1684 :
1685 : /**
1686 : * Check if a block position in a disk is deleted.
1687 : */
1688 4529621 : static int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
1689 : {
1690 4529621 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1691 :
1692 4529621 : return block_state_get(block) == BLOCK_STATE_DELETED;
1693 : }
1694 :
1695 : /**
1696 : * Flush the file checking the final CRC.
1697 : * We exploit the fact that the CRC is always stored in the last 4 bytes.
1698 : */
1699 1 : static void decoding_error(const char* path, STREAM* f)
1700 : {
1701 : unsigned char buf[4];
1702 : uint32_t crc_stored;
1703 : uint32_t crc_computed;
1704 :
1705 1 : if (seof(f)) {
1706 : /* LCOV_EXCL_START */
1707 : log_fatal("Unexpected end of content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1708 : log_fatal("This content file is truncated. Use an alternate copy.\n");
1709 : exit(EXIT_FAILURE);
1710 : /* LCOV_EXCL_STOP */
1711 : }
1712 :
1713 1 : if (serror(f)) {
1714 : /* LCOV_EXCL_START */
1715 : log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1716 : exit(EXIT_FAILURE);
1717 : /* LCOV_EXCL_STOP */
1718 : }
1719 :
1720 1 : log_fatal("Decoding error in '%s' at offset %" PRIi64 "\n", path, stell(f));
1721 :
1722 1 : if (sdeplete(f, buf) != 0) {
1723 : /* LCOV_EXCL_START */
1724 : log_fatal("Error flushing the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1725 : exit(EXIT_FAILURE);
1726 : /* LCOV_EXCL_STOP */
1727 : }
1728 :
1729 : /* get the stored crc from the last four bytes */
1730 1 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
1731 :
1732 : /* get the computed crc */
1733 1 : crc_computed = scrc(f);
1734 :
1735 : /* adjust the stored crc to include itself */
1736 1 : crc_stored = crc32c(crc_stored, buf, 4);
1737 :
1738 1 : if (crc_computed != crc_stored) {
1739 1 : log_fatal("Mismatching CRC in '%s'\n", path);
1740 1 : log_fatal("This content file is damaged! Use an alternate copy.\n");
1741 1 : exit(EXIT_FAILURE);
1742 : } else {
1743 0 : log_fatal("The file CRC is correct!\n");
1744 : }
1745 0 : }
1746 :
1747 254 : static void state_read_content(struct snapraid_state* state, const char* path, STREAM* f)
1748 : {
1749 : block_off_t blockmax;
1750 : unsigned count_file;
1751 : unsigned count_hardlink;
1752 : unsigned count_symlink;
1753 : unsigned count_dir;
1754 : int crc_checked;
1755 : char buffer[PATH_MAX];
1756 : int ret;
1757 : tommy_array disk_mapping;
1758 : uint32_t mapping_max;
1759 :
1760 254 : blockmax = 0;
1761 254 : count_file = 0;
1762 254 : count_hardlink = 0;
1763 254 : count_symlink = 0;
1764 254 : count_dir = 0;
1765 254 : crc_checked = 0;
1766 254 : mapping_max = 0;
1767 254 : tommy_array_init(&disk_mapping);
1768 :
1769 254 : ret = sread(f, buffer, 12);
1770 254 : if (ret < 0) {
1771 : /* LCOV_EXCL_START */
1772 : decoding_error(path, f);
1773 : log_fatal("Invalid header!\n");
1774 : os_abort();
1775 : /* LCOV_EXCL_STOP */
1776 : }
1777 :
1778 : /*
1779 : * File format versions:
1780 : * - SNAPCNT1/SnapRAID 4.0 First version.
1781 : * - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support.
1782 : * The previous 'm' entry is now deprecated, but supported for importing.
1783 : * Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
1784 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
1785 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
1786 : * The previous 'P' entry is now deprecated, but supported for importing.
1787 : */
1788 254 : if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
1789 254 : && memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
1790 254 : && memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
1791 : ) {
1792 : /* LCOV_EXCL_START */
1793 : if (memcmp(buffer, "SNAPCNT", 7) != 0) {
1794 : decoding_error(path, f);
1795 : log_fatal("Invalid header!\n");
1796 : os_abort();
1797 : } else {
1798 : log_fatal("The content file '%s' was generated with a newer version of SnapRAID!\n", path);
1799 : exit(EXIT_FAILURE);
1800 : }
1801 : /* LCOV_EXCL_STOP */
1802 : }
1803 :
1804 : while (1) {
1805 : int c;
1806 :
1807 : /* read the command */
1808 3735613 : c = sgetc(f);
1809 3735613 : if (c == EOF) {
1810 253 : break;
1811 : }
1812 :
1813 3735360 : if (c == 'f') {
1814 : /* file */
1815 : char sub[PATH_MAX];
1816 : uint64_t v_size;
1817 : uint64_t v_mtime_sec;
1818 : uint32_t v_mtime_nsec;
1819 : uint64_t v_inode;
1820 : uint32_t v_idx;
1821 : struct snapraid_file* file;
1822 : struct snapraid_disk* disk;
1823 : uint32_t mapping;
1824 :
1825 3611762 : ret = sgetb32(f, &mapping);
1826 3611762 : if (ret < 0 || mapping >= mapping_max) {
1827 : /* LCOV_EXCL_START */
1828 : decoding_error(path, f);
1829 : log_fatal("Internal inconsistency in mapping index!\n");
1830 : os_abort();
1831 : /* LCOV_EXCL_STOP */
1832 : }
1833 3611762 : disk = tommy_array_get(&disk_mapping, mapping);
1834 :
1835 3611762 : ret = sgetb64(f, &v_size);
1836 3611762 : if (ret < 0) {
1837 : /* LCOV_EXCL_START */
1838 : decoding_error(path, f);
1839 : os_abort();
1840 : /* LCOV_EXCL_STOP */
1841 : }
1842 :
1843 3611762 : if (state->block_size == 0) {
1844 : /* LCOV_EXCL_START */
1845 : decoding_error(path, f);
1846 : log_fatal("Internal incosistency due zero blocksize!\n");
1847 : exit(EXIT_FAILURE);
1848 : /* LCOV_EXCL_STOP */
1849 : }
1850 :
1851 : /* check for impossible file size to avoid to crash for a too big allocation */
1852 3611762 : if (v_size / state->block_size > blockmax) {
1853 : /* LCOV_EXCL_START */
1854 : decoding_error(path, f);
1855 : log_fatal("Internal inconsistency in file size too big!\n");
1856 : os_abort();
1857 : /* LCOV_EXCL_STOP */
1858 : }
1859 :
1860 3611762 : ret = sgetb64(f, &v_mtime_sec);
1861 3611762 : if (ret < 0) {
1862 : /* LCOV_EXCL_START */
1863 : decoding_error(path, f);
1864 : os_abort();
1865 : /* LCOV_EXCL_STOP */
1866 : }
1867 :
1868 3611762 : ret = sgetb32(f, &v_mtime_nsec);
1869 3611762 : if (ret < 0) {
1870 : /* LCOV_EXCL_START */
1871 : decoding_error(path, f);
1872 : os_abort();
1873 : /* LCOV_EXCL_STOP */
1874 : }
1875 :
1876 : /* STAT_NSEC_INVALID is encoded as 0 */
1877 3611762 : if (v_mtime_nsec == 0)
1878 0 : v_mtime_nsec = STAT_NSEC_INVALID;
1879 : else
1880 3611762 : --v_mtime_nsec;
1881 :
1882 3611762 : ret = sgetb64(f, &v_inode);
1883 3611762 : if (ret < 0) {
1884 : /* LCOV_EXCL_START */
1885 : decoding_error(path, f);
1886 : os_abort();
1887 : /* LCOV_EXCL_STOP */
1888 : }
1889 :
1890 3611762 : ret = sgetbs(f, sub, sizeof(sub));
1891 3611762 : if (ret < 0) {
1892 : /* LCOV_EXCL_START */
1893 : decoding_error(path, f);
1894 : os_abort();
1895 : /* LCOV_EXCL_STOP */
1896 : }
1897 3611762 : if (!*sub) {
1898 : /* LCOV_EXCL_START */
1899 : decoding_error(path, f);
1900 : log_fatal("Internal inconsistency for null file!\n");
1901 : os_abort();
1902 : /* LCOV_EXCL_STOP */
1903 : }
1904 :
1905 : /* allocate the file */
1906 3611762 : file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0);
1907 :
1908 : /* insert the file in the file containers */
1909 3611762 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
1910 3611762 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
1911 3611762 : tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
1912 3611762 : tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
1913 :
1914 : /* read all the blocks */
1915 3611762 : v_idx = 0;
1916 10855984 : while (v_idx < file->blockmax) {
1917 : block_off_t v_pos;
1918 : uint32_t v_count;
1919 :
1920 : /* get the "subcommand */
1921 3632460 : c = sgetc(f);
1922 :
1923 3632460 : ret = sgetb32(f, &v_pos);
1924 3632460 : if (ret < 0) {
1925 : /* LCOV_EXCL_START */
1926 : decoding_error(path, f);
1927 : os_abort();
1928 : /* LCOV_EXCL_STOP */
1929 : }
1930 :
1931 3632460 : ret = sgetb32(f, &v_count);
1932 3632460 : if (ret < 0) {
1933 : /* LCOV_EXCL_START */
1934 : decoding_error(path, f);
1935 : os_abort();
1936 : /* LCOV_EXCL_STOP */
1937 : }
1938 :
1939 3632460 : if (v_idx + v_count > file->blockmax) {
1940 : /* LCOV_EXCL_START */
1941 : decoding_error(path, f);
1942 : log_fatal("Internal inconsistency in block number!\n");
1943 : os_abort();
1944 : /* LCOV_EXCL_STOP */
1945 : }
1946 :
1947 3632460 : if (v_pos + v_count > blockmax) {
1948 : /* LCOV_EXCL_START */
1949 : decoding_error(path, f);
1950 : log_fatal("Internal inconsistency in block size %u/%u!\n", blockmax, v_pos + v_count);
1951 : os_abort();
1952 : /* LCOV_EXCL_START */
1953 : }
1954 :
1955 : /* fill the blocks in the run */
1956 : while (v_count) {
1957 : struct snapraid_block* block = fs_file2block_get(file, v_idx);
1958 :
1959 : switch (c) {
1960 : case 'b' :
1961 : block_state_set(block, BLOCK_STATE_BLK);
1962 : break;
1963 : case 'n' :
1964 : /* deprecated NEW blocks are converted to CHG ones */
1965 : block_state_set(block, BLOCK_STATE_CHG);
1966 : break;
1967 : case 'g' :
1968 : block_state_set(block, BLOCK_STATE_CHG);
1969 : break;
1970 : case 'p' :
1971 : block_state_set(block, BLOCK_STATE_REP);
1972 : break;
1973 : default :
1974 : /* LCOV_EXCL_START */
1975 : decoding_error(path, f);
1976 : log_fatal("Invalid block type!\n");
1977 : os_abort();
1978 : /* LCOV_EXCL_STOP */
1979 : }
1980 :
1981 : /* read the hash only for 'blk/chg/rep', and not for 'new' */
1982 8896100 : if (c != 'n') {
1983 8896100 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
1984 8896100 : if (ret < 0) {
1985 : /* LCOV_EXCL_START */
1986 : decoding_error(path, f);
1987 : os_abort();
1988 : /* LCOV_EXCL_STOP */
1989 : }
1990 : } else {
1991 : /* set the ZERO hash for deprecated NEW blocks */
1992 0 : hash_zero_set(block->hash);
1993 : }
1994 :
1995 : /* if the block contains a hash of past data */
1996 : /* and we are clearing such indeterminate hashes */
1997 8896100 : if (state->clear_past_hash
1998 2542004 : && block_has_past_hash(block)
1999 : ) {
2000 : /* set the hash value to INVALID */
2001 28006 : hash_invalid_set(block->hash);
2002 : }
2003 :
2004 : /* if we are disabling the copy optimization */
2005 : /* we want also to clear any already previously stored information */
2006 : /* in other sync commands */
2007 : /* note that this is required only in sync, and we detect */
2008 : /* this using the clear_past_hash flag */
2009 8896100 : if (state->clear_past_hash
2010 2542004 : && state->opt.force_nocopy
2011 14195 : && block_state_get(block) == BLOCK_STATE_REP
2012 : ) {
2013 : /* set the hash value to INVALID */
2014 1 : hash_invalid_set(block->hash);
2015 : /* convert from REP to CHG block */
2016 1 : block_state_set(block, BLOCK_STATE_CHG);
2017 : }
2018 :
2019 : /* if we want a full reallocation, marks block as invalid parity */
2020 : /* note that we do this after the force_nocopy option */
2021 : /* to avoid to mixup the two things */
2022 8896100 : if (state->opt.force_realloc
2023 27826 : && block_state_get(block) == BLOCK_STATE_BLK) {
2024 : /* convert from BLK to REP */
2025 27826 : block_state_set(block, BLOCK_STATE_REP);
2026 : }
2027 :
2028 : /* set the parity association */
2029 8896100 : fs_allocate(disk, v_pos, file, v_idx);
2030 :
2031 : /* go to the next block */
2032 8896100 : ++v_idx;
2033 8896100 : ++v_pos;
2034 8896100 : --v_count;
2035 : }
2036 : }
2037 :
2038 : /* stat */
2039 3611762 : ++count_file;
2040 123598 : } else if (c == 'i') {
2041 : /* "inf" command */
2042 : snapraid_info info;
2043 : uint32_t v_pos;
2044 : uint32_t v_oldest;
2045 :
2046 253 : ret = sgetb32(f, &v_oldest);
2047 253 : if (ret < 0) {
2048 : /* LCOV_EXCL_START */
2049 : decoding_error(path, f);
2050 : os_abort();
2051 : /* LCOV_EXCL_STOP */
2052 : }
2053 :
2054 253 : v_pos = 0;
2055 55856 : while (v_pos < blockmax) {
2056 : int bad;
2057 : int rehash;
2058 : int justsynced;
2059 : uint32_t t;
2060 : uint32_t flag;
2061 : uint32_t v_count;
2062 :
2063 55350 : ret = sgetb32(f, &v_count);
2064 55350 : if (ret < 0) {
2065 : /* LCOV_EXCL_START */
2066 : decoding_error(path, f);
2067 : os_abort();
2068 : /* LCOV_EXCL_STOP */
2069 : }
2070 :
2071 55350 : if (v_pos + v_count > blockmax) {
2072 : /* LCOV_EXCL_START */
2073 : decoding_error(path, f);
2074 : log_fatal("Internal inconsistency in info size %u/%u!\n", blockmax, v_pos + v_count);
2075 : os_abort();
2076 : /* LCOV_EXCL_STOP */
2077 : }
2078 :
2079 55350 : ret = sgetb32(f, &flag);
2080 55350 : if (ret < 0) {
2081 : /* LCOV_EXCL_START */
2082 : decoding_error(path, f);
2083 : os_abort();
2084 : /* LCOV_EXCL_STOP */
2085 : }
2086 :
2087 : /* if there is an info */
2088 55350 : if ((flag & 1) != 0) {
2089 : /* read the time */
2090 55335 : ret = sgetb32(f, &t);
2091 55335 : if (ret < 0) {
2092 : /* LCOV_EXCL_START */
2093 : decoding_error(path, f);
2094 : os_abort();
2095 : /* LCOV_EXCL_STOP */
2096 : }
2097 :
2098 : /* analyze the flags */
2099 55335 : bad = (flag & 2) != 0;
2100 55335 : rehash = (flag & 4) != 0;
2101 55335 : justsynced = (flag & 8) != 0;
2102 :
2103 55335 : if (rehash && state->prevhash == HASH_UNDEFINED) {
2104 : /* LCOV_EXCL_START */
2105 : decoding_error(path, f);
2106 : log_fatal("Internal inconsistency for missing previous checksum!\n");
2107 : os_abort();
2108 : /* LCOV_EXCL_STOP */
2109 : }
2110 :
2111 55335 : info = info_make(t + v_oldest, bad, rehash, justsynced);
2112 : } else {
2113 15 : info = 0;
2114 : }
2115 :
2116 1658623 : while (v_count) {
2117 : /* insert the info in the array */
2118 1547923 : info_set(&state->infoarr, v_pos, info);
2119 :
2120 : /* ensure that an info is present only for used positions */
2121 1547923 : if (fs_info_is_required(state, v_pos)) {
2122 1489158 : if (!info) {
2123 : /* LCOV_EXCL_START */
2124 : decoding_error(path, f);
2125 : log_fatal("Internal inconsistency for missing info!\n");
2126 : os_abort();
2127 : /* LCOV_EXCL_STOP */
2128 : }
2129 : } else {
2130 : /* extra info are accepted for backward compatibility */
2131 : /* they are discarded at the first write */
2132 : }
2133 :
2134 : /* go to next block */
2135 1547923 : ++v_pos;
2136 1547923 : --v_count;
2137 : }
2138 : }
2139 123345 : } else if (c == 'h') {
2140 : /* hole */
2141 : uint32_t v_pos;
2142 : struct snapraid_disk* disk;
2143 : uint32_t mapping;
2144 :
2145 1397 : ret = sgetb32(f, &mapping);
2146 1397 : if (ret < 0 || mapping >= mapping_max) {
2147 : /* LCOV_EXCL_START */
2148 : decoding_error(path, f);
2149 : log_fatal("Internal inconsistency in mapping index!\n");
2150 : os_abort();
2151 : /* LCOV_EXCL_STOP */
2152 : }
2153 1397 : disk = tommy_array_get(&disk_mapping, mapping);
2154 :
2155 1397 : v_pos = 0;
2156 4257 : while (v_pos < blockmax) {
2157 : uint32_t v_idx;
2158 : uint32_t v_count;
2159 : struct snapraid_file* deleted;
2160 :
2161 1463 : ret = sgetb32(f, &v_count);
2162 1463 : if (ret < 0) {
2163 : /* LCOV_EXCL_START */
2164 : decoding_error(path, f);
2165 : os_abort();
2166 : /* LCOV_EXCL_STOP */
2167 : }
2168 :
2169 1463 : if (v_pos + v_count > blockmax) {
2170 : /* LCOV_EXCL_START */
2171 : decoding_error(path, f);
2172 : log_fatal("Internal inconsistency in hole size %u/%u!\n", blockmax, v_pos + v_count);
2173 : os_abort();
2174 : /* LCOV_EXCL_STOP */
2175 : }
2176 :
2177 : /* get the sub-command */
2178 1463 : c = sgetc(f);
2179 :
2180 1463 : switch (c) {
2181 : case 'o' :
2182 : /* if it's a run of deleted blocks */
2183 :
2184 : /* allocate a fake deleted file */
2185 35 : deleted = file_alloc(state->block_size, "<deleted>", v_count * (data_off_t)state->block_size, 0, 0, 0, 0);
2186 :
2187 : /* mark the file as deleted */
2188 35 : file_flag_set(deleted, FILE_IS_DELETED);
2189 :
2190 : /* insert it in the list of deleted files */
2191 35 : tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted);
2192 :
2193 : /* process all blocks */
2194 35 : v_idx = 0;
2195 27956 : while (v_count) {
2196 27886 : struct snapraid_block* block = fs_file2block_get(deleted, v_idx);
2197 :
2198 : /* set the block as deleted */
2199 27886 : block_state_set(block, BLOCK_STATE_DELETED);
2200 :
2201 : /* read the hash */
2202 27886 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
2203 27886 : if (ret < 0) {
2204 : /* LCOV_EXCL_START */
2205 : decoding_error(path, f);
2206 : os_abort();
2207 : /* LCOV_EXCL_STOP */
2208 : }
2209 :
2210 : /* if we are clearing indeterminate hashes */
2211 27886 : if (state->clear_past_hash) {
2212 : /* set the hash value to INVALID */
2213 9432 : hash_invalid_set(block->hash);
2214 : }
2215 :
2216 : /* insert the block in the block array */
2217 27886 : fs_allocate(disk, v_pos, deleted, v_idx);
2218 :
2219 : /* go to next block */
2220 27886 : ++v_pos;
2221 27886 : ++v_idx;
2222 27886 : --v_count;
2223 : }
2224 35 : break;
2225 : case 'O' :
2226 : /* go to the next run */
2227 1428 : v_pos += v_count;
2228 1428 : break;
2229 : default :
2230 : /* LCOV_EXCL_START */
2231 : decoding_error(path, f);
2232 : log_fatal("Invalid hole type!\n");
2233 : os_abort();
2234 : /* LCOV_EXCL_STOP */
2235 : }
2236 : }
2237 121948 : } else if (c == 's') {
2238 : /* symlink */
2239 : char sub[PATH_MAX];
2240 : char linkto[PATH_MAX];
2241 : struct snapraid_link* slink;
2242 : struct snapraid_disk* disk;
2243 : uint32_t mapping;
2244 :
2245 116468 : ret = sgetb32(f, &mapping);
2246 116468 : if (ret < 0 || mapping >= mapping_max) {
2247 : /* LCOV_EXCL_START */
2248 : decoding_error(path, f);
2249 : log_fatal("Internal inconsistency in mapping index!\n");
2250 : os_abort();
2251 : /* LCOV_EXCL_STOP */
2252 : }
2253 116468 : disk = tommy_array_get(&disk_mapping, mapping);
2254 :
2255 116468 : ret = sgetbs(f, sub, sizeof(sub));
2256 116468 : if (ret < 0) {
2257 : /* LCOV_EXCL_START */
2258 : decoding_error(path, f);
2259 : os_abort();
2260 : /* LCOV_EXCL_STOP */
2261 : }
2262 :
2263 116468 : if (!*sub) {
2264 : /* LCOV_EXCL_START */
2265 : decoding_error(path, f);
2266 : log_fatal("Internal inconsistency for null symlink!\n");
2267 : os_abort();
2268 : /* LCOV_EXCL_STOP */
2269 : }
2270 :
2271 116468 : ret = sgetbs(f, linkto, sizeof(linkto));
2272 116468 : if (ret < 0) {
2273 : /* LCOV_EXCL_START */
2274 : decoding_error(path, f);
2275 : os_abort();
2276 : /* LCOV_EXCL_STOP */
2277 : }
2278 :
2279 : /* allocate the link as symbolic link */
2280 116468 : slink = link_alloc(sub, linkto, FILE_IS_SYMLINK);
2281 :
2282 : /* insert the link in the link containers */
2283 116468 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2284 116468 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2285 :
2286 : /* stat */
2287 116468 : ++count_symlink;
2288 5480 : } else if (c == 'a') {
2289 : /* hardlink */
2290 : char sub[PATH_MAX];
2291 : char linkto[PATH_MAX];
2292 : struct snapraid_link* slink;
2293 : struct snapraid_disk* disk;
2294 : uint32_t mapping;
2295 :
2296 597 : ret = sgetb32(f, &mapping);
2297 597 : if (ret < 0 || mapping >= mapping_max) {
2298 : /* LCOV_EXCL_START */
2299 : decoding_error(path, f);
2300 : log_fatal("Internal inconsistency in mapping index!\n");
2301 : os_abort();
2302 : /* LCOV_EXCL_STOP */
2303 : }
2304 597 : disk = tommy_array_get(&disk_mapping, mapping);
2305 :
2306 597 : ret = sgetbs(f, sub, sizeof(sub));
2307 597 : if (ret < 0) {
2308 : /* LCOV_EXCL_START */
2309 : decoding_error(path, f);
2310 : os_abort();
2311 : /* LCOV_EXCL_STOP */
2312 : }
2313 :
2314 597 : if (!*sub) {
2315 : /* LCOV_EXCL_START */
2316 : decoding_error(path, f);
2317 : log_fatal("Internal inconsistency for null hardlink!\n");
2318 : os_abort();
2319 : /* LCOV_EXCL_STOP */
2320 : }
2321 :
2322 597 : ret = sgetbs(f, linkto, sizeof(linkto));
2323 597 : if (ret < 0) {
2324 : /* LCOV_EXCL_START */
2325 : decoding_error(path, f);
2326 : os_abort();
2327 : /* LCOV_EXCL_STOP */
2328 : }
2329 :
2330 597 : if (!*linkto) {
2331 : /* LCOV_EXCL_START */
2332 : decoding_error(path, f);
2333 : log_fatal("Internal inconsistency for empty hardlink '%s'!\n", sub);
2334 : os_abort();
2335 : /* LCOV_EXCL_STOP */
2336 : }
2337 :
2338 : /* allocate the link as hard link */
2339 597 : slink = link_alloc(sub, linkto, FILE_IS_HARDLINK);
2340 :
2341 : /* insert the link in the link containers */
2342 597 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2343 597 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2344 :
2345 : /* stat */
2346 597 : ++count_hardlink;
2347 4883 : } else if (c == 'r') {
2348 : /* dir */
2349 : char sub[PATH_MAX];
2350 : struct snapraid_dir* dir;
2351 : struct snapraid_disk* disk;
2352 : uint32_t mapping;
2353 :
2354 790 : ret = sgetb32(f, &mapping);
2355 790 : if (ret < 0 || mapping >= mapping_max) {
2356 : /* LCOV_EXCL_START */
2357 : decoding_error(path, f);
2358 : log_fatal("Internal inconsistency in mapping index!\n");
2359 : os_abort();
2360 : /* LCOV_EXCL_STOP */
2361 : }
2362 790 : disk = tommy_array_get(&disk_mapping, mapping);
2363 :
2364 790 : ret = sgetbs(f, sub, sizeof(sub));
2365 790 : if (ret < 0) {
2366 : /* LCOV_EXCL_START */
2367 : decoding_error(path, f);
2368 : os_abort();
2369 : /* LCOV_EXCL_STOP */
2370 : }
2371 :
2372 790 : if (!*sub) {
2373 : /* LCOV_EXCL_START */
2374 : decoding_error(path, f);
2375 : log_fatal("Internal inconsistency for null dir!\n");
2376 : os_abort();
2377 : /* LCOV_EXCL_STOP */
2378 : }
2379 :
2380 : /* allocate the dir */
2381 790 : dir = dir_alloc(sub);
2382 :
2383 : /* insert the dir in the dir containers */
2384 790 : tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
2385 790 : tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
2386 :
2387 : /* stat */
2388 790 : ++count_dir;
2389 4093 : } else if (c == 'c') {
2390 : /* get the subcommand */
2391 254 : c = sgetc(f);
2392 :
2393 254 : switch (c) {
2394 : case 'u' :
2395 166 : state->hash = HASH_MURMUR3;
2396 166 : break;
2397 : case 'k' :
2398 88 : state->hash = HASH_SPOOKY2;
2399 88 : break;
2400 : default :
2401 : /* LCOV_EXCL_START */
2402 : decoding_error(path, f);
2403 : log_fatal("Invalid checksum!\n");
2404 : os_abort();
2405 : /* LCOV_EXCL_STOP */
2406 : }
2407 :
2408 : /* read the seed */
2409 254 : ret = sread(f, state->hashseed, HASH_MAX);
2410 254 : if (ret < 0) {
2411 : /* LCOV_EXCL_START */
2412 : decoding_error(path, f);
2413 : os_abort();
2414 : /* LCOV_EXCL_STOP */
2415 : }
2416 3839 : } else if (c == 'C') {
2417 : /* get the sub-command */
2418 4 : c = sgetc(f);
2419 :
2420 4 : switch (c) {
2421 : case 'u' :
2422 4 : state->prevhash = HASH_MURMUR3;
2423 4 : break;
2424 : case 'k' :
2425 0 : state->prevhash = HASH_SPOOKY2;
2426 0 : break;
2427 : default :
2428 : /* LCOV_EXCL_START */
2429 : decoding_error(path, f);
2430 : log_fatal("Invalid checksum!\n");
2431 : os_abort();
2432 : /* LCOV_EXCL_STOP */
2433 : }
2434 :
2435 : /* read the seed */
2436 4 : ret = sread(f, state->prevhashseed, HASH_MAX);
2437 4 : if (ret < 0) {
2438 : /* LCOV_EXCL_START */
2439 : decoding_error(path, f);
2440 : os_abort();
2441 : /* LCOV_EXCL_STOP */
2442 : }
2443 3835 : } else if (c == 'z') {
2444 : uint32_t block_size;
2445 :
2446 254 : ret = sgetb32(f, &block_size);
2447 254 : if (ret < 0) {
2448 : /* LCOV_EXCL_START */
2449 : decoding_error(path, f);
2450 : os_abort();
2451 : /* LCOV_EXCL_STOP */
2452 : }
2453 :
2454 254 : if (block_size == 0) {
2455 : /* LCOV_EXCL_START */
2456 : decoding_error(path, f);
2457 : log_fatal("Zero 'blocksize' specification in the content file!\n");
2458 : exit(EXIT_FAILURE);
2459 : /* LCOV_EXCL_STOP */
2460 : }
2461 :
2462 : /* without configuration, auto assign the block size */
2463 254 : if (state->no_conf) {
2464 1 : state->block_size = block_size;
2465 : }
2466 :
2467 254 : if (block_size != state->block_size) {
2468 : /* LCOV_EXCL_START */
2469 : decoding_error(path, f);
2470 : log_fatal("Mismatching 'blocksize' specification in the content file!\n");
2471 : log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", block_size / KIBI);
2472 : exit(EXIT_FAILURE);
2473 : /* LCOV_EXCL_STOP */
2474 : }
2475 3581 : } else if (c == 'y') {
2476 : uint32_t hash_size;
2477 :
2478 254 : ret = sgetb32(f, &hash_size);
2479 254 : if (ret < 0) {
2480 : /* LCOV_EXCL_START */
2481 : decoding_error(path, f);
2482 : os_abort();
2483 : /* LCOV_EXCL_STOP */
2484 : }
2485 :
2486 254 : if (hash_size < 2 || hash_size > HASH_MAX) {
2487 : /* LCOV_EXCL_START */
2488 : decoding_error(path, f);
2489 : log_fatal("Invalid 'hashsize' specification in the content file!\n");
2490 : exit(EXIT_FAILURE);
2491 : /* LCOV_EXCL_STOP */
2492 : }
2493 :
2494 : /* without configuration, auto assign the block size */
2495 254 : if (state->no_conf) {
2496 1 : BLOCK_HASH_SIZE = hash_size;
2497 : }
2498 :
2499 254 : if ((int)hash_size != BLOCK_HASH_SIZE) {
2500 : /* LCOV_EXCL_START */
2501 : decoding_error(path, f);
2502 : log_fatal("Mismatching 'hashsize' specification in the content file!\n");
2503 : log_fatal("Please restore the 'hashsize' value in the configuration file to '%u'\n", hash_size);
2504 : exit(EXIT_FAILURE);
2505 : /* LCOV_EXCL_STOP */
2506 : }
2507 3327 : } else if (c == 'x') {
2508 254 : ret = sgetb32(f, &blockmax);
2509 254 : if (ret < 0) {
2510 : /* LCOV_EXCL_START */
2511 : decoding_error(path, f);
2512 : os_abort();
2513 : /* LCOV_EXCL_STOP */
2514 : }
2515 4476 : } else if (c == 'm' || c == 'M') {
2516 : struct snapraid_map* map;
2517 : char uuid[UUID_MAX];
2518 : uint32_t v_pos;
2519 : uint32_t v_total_blocks;
2520 : uint32_t v_free_blocks;
2521 : struct snapraid_disk* disk;
2522 :
2523 1403 : ret = sgetbs(f, buffer, sizeof(buffer));
2524 1403 : if (ret < 0) {
2525 : /* LCOV_EXCL_START */
2526 : decoding_error(path, f);
2527 : os_abort();
2528 : /* LCOV_EXCL_STOP */
2529 : }
2530 :
2531 1403 : ret = sgetb32(f, &v_pos);
2532 1403 : if (ret < 0) {
2533 : /* LCOV_EXCL_START */
2534 : decoding_error(path, f);
2535 : os_abort();
2536 : /* LCOV_EXCL_STOP */
2537 : }
2538 :
2539 : /* from SnapRAID 7.0 the 'M' command includes the free space */
2540 1403 : if (c == 'M') {
2541 1403 : ret = sgetb32(f, &v_total_blocks);
2542 1403 : if (ret < 0) {
2543 : /* LCOV_EXCL_START */
2544 : decoding_error(path, f);
2545 : os_abort();
2546 : /* LCOV_EXCL_STOP */
2547 : }
2548 :
2549 1403 : ret = sgetb32(f, &v_free_blocks);
2550 1403 : if (ret < 0) {
2551 : /* LCOV_EXCL_START */
2552 : decoding_error(path, f);
2553 : os_abort();
2554 : /* LCOV_EXCL_STOP */
2555 : }
2556 : } else {
2557 0 : v_total_blocks = 0;
2558 0 : v_free_blocks = 0;
2559 : }
2560 :
2561 : /* read the uuid */
2562 1403 : ret = sgetbs(f, uuid, sizeof(uuid));
2563 1403 : if (ret < 0) {
2564 : /* LCOV_EXCL_START */
2565 : decoding_error(path, f);
2566 : os_abort();
2567 : /* LCOV_EXCL_STOP */
2568 : }
2569 :
2570 : /* find the disk */
2571 1403 : disk = find_disk_by_name(state, buffer);
2572 1403 : if (!disk) {
2573 : /* search by UUID if renamed */
2574 2 : disk = find_disk_by_uuid(state, uuid);
2575 2 : if (disk) {
2576 2 : log_fatal("WARNING! Renaming disk '%s' to '%s'\n", buffer, disk->name);
2577 :
2578 : /* write the new state with the new name */
2579 2 : state->need_write = 1;
2580 : }
2581 : }
2582 1403 : if (!disk) {
2583 : /* LCOV_EXCL_START */
2584 : decoding_error(path, f);
2585 : log_fatal("Disk '%s' with uuid '%s' not present in the configuration file!\n", buffer, uuid);
2586 : log_fatal("If you have removed it from the configuration file, please restore it\n");
2587 : /* if it's a command without UUID, it cannot autorename using UUID */
2588 : if (state->opt.skip_disk_access)
2589 : log_fatal("If you have renamed it, run 'sync' to update the new name\n");
2590 : exit(EXIT_FAILURE);
2591 : /* LCOV_EXCL_STOP */
2592 : }
2593 :
2594 1403 : map = map_alloc(disk->name, v_pos, v_total_blocks, v_free_blocks, uuid);
2595 :
2596 1403 : tommy_list_insert_tail(&state->maplist, &map->node, map);
2597 :
2598 : /* insert in the mapping vector */
2599 1403 : tommy_array_grow(&disk_mapping, mapping_max + 1);
2600 1403 : tommy_array_set(&disk_mapping, mapping_max, disk);
2601 1403 : ++mapping_max;
2602 1670 : } else if (c == 'P') {
2603 : /* from SnapRAID 7.0 the 'P' command includes the free space */
2604 : /* from SnapRAID 11.0 the 'P' command is deprecated by 'Q' */
2605 : char v_uuid[UUID_MAX];
2606 : uint32_t v_level;
2607 : uint32_t v_total_blocks;
2608 : uint32_t v_free_blocks;
2609 :
2610 0 : ret = sgetb32(f, &v_level);
2611 0 : if (ret < 0) {
2612 : /* LCOV_EXCL_START */
2613 : decoding_error(path, f);
2614 : os_abort();
2615 : /* LCOV_EXCL_STOP */
2616 : }
2617 :
2618 0 : ret = sgetb32(f, &v_total_blocks);
2619 0 : if (ret < 0) {
2620 : /* LCOV_EXCL_START */
2621 : decoding_error(path, f);
2622 : os_abort();
2623 : /* LCOV_EXCL_STOP */
2624 : }
2625 :
2626 0 : ret = sgetb32(f, &v_free_blocks);
2627 0 : if (ret < 0) {
2628 : /* LCOV_EXCL_START */
2629 : decoding_error(path, f);
2630 : os_abort();
2631 : /* LCOV_EXCL_STOP */
2632 : }
2633 :
2634 0 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2635 0 : if (ret < 0) {
2636 : /* LCOV_EXCL_START */
2637 : decoding_error(path, f);
2638 : os_abort();
2639 : /* LCOV_EXCL_STOP */
2640 : }
2641 :
2642 0 : if (v_level >= LEV_MAX) {
2643 : /* LCOV_EXCL_START */
2644 : decoding_error(path, f);
2645 : log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2646 : exit(EXIT_FAILURE);
2647 : /* LCOV_EXCL_STOP */
2648 : }
2649 :
2650 : /* auto configure if configuration is missing */
2651 0 : if (state->no_conf) {
2652 0 : if (v_level >= state->level)
2653 0 : state->level = v_level + 1;
2654 : }
2655 :
2656 : /* if we use this parity entry */
2657 0 : if (v_level < state->level) {
2658 : /* if the configuration has more splits, keep them */
2659 0 : if (state->parity[v_level].split_mac < 1)
2660 0 : state->parity[v_level].split_mac = 1;
2661 : /* set the parity info */
2662 0 : pathcpy(state->parity[v_level].split_map[0].uuid, sizeof(state->parity[v_level].split_map[0].uuid), v_uuid);
2663 0 : state->parity[v_level].total_blocks = v_total_blocks;
2664 0 : state->parity[v_level].free_blocks = v_free_blocks;
2665 : }
2666 1670 : } else if (c == 'Q') {
2667 : /* from SnapRAID 11.0 the 'Q' command include size info and multi file support */
2668 : uint32_t v_level;
2669 : uint32_t v_total_blocks;
2670 : uint32_t v_free_blocks;
2671 : uint32_t v_split_mac;
2672 : unsigned s;
2673 :
2674 1417 : ret = sgetb32(f, &v_level);
2675 1417 : if (ret < 0) {
2676 : /* LCOV_EXCL_START */
2677 : decoding_error(path, f);
2678 : os_abort();
2679 : /* LCOV_EXCL_STOP */
2680 : }
2681 :
2682 1417 : ret = sgetb32(f, &v_total_blocks);
2683 1417 : if (ret < 0) {
2684 : /* LCOV_EXCL_START */
2685 : decoding_error(path, f);
2686 : os_abort();
2687 : /* LCOV_EXCL_STOP */
2688 : }
2689 :
2690 1417 : ret = sgetb32(f, &v_free_blocks);
2691 1417 : if (ret < 0) {
2692 : /* LCOV_EXCL_START */
2693 : decoding_error(path, f);
2694 : os_abort();
2695 : /* LCOV_EXCL_STOP */
2696 : }
2697 :
2698 1417 : ret = sgetb32(f, &v_split_mac);
2699 1417 : if (ret < 0) {
2700 : /* LCOV_EXCL_START */
2701 : decoding_error(path, f);
2702 : os_abort();
2703 : /* LCOV_EXCL_STOP */
2704 : }
2705 :
2706 1417 : if (v_level >= LEV_MAX) {
2707 : /* LCOV_EXCL_START */
2708 : decoding_error(path, f);
2709 : log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2710 : exit(EXIT_FAILURE);
2711 : /* LCOV_EXCL_STOP */
2712 : }
2713 :
2714 : /* auto configure if configuration is missing */
2715 1417 : if (state->no_conf) {
2716 6 : if (v_level >= state->level)
2717 5 : state->level = v_level + 1;
2718 6 : if (state->parity[v_level].split_mac < v_split_mac)
2719 6 : state->parity[v_level].split_mac = v_split_mac;
2720 : }
2721 :
2722 : /* if we use this parity entry */
2723 1417 : if (v_level < state->level) {
2724 : /* set the parity info */
2725 1158 : state->parity[v_level].total_blocks = v_total_blocks;
2726 1158 : state->parity[v_level].free_blocks = v_free_blocks;
2727 : }
2728 :
2729 7083 : for (s = 0; s < v_split_mac; ++s) {
2730 : char v_path[PATH_MAX];
2731 : char v_uuid[UUID_MAX];
2732 : uint64_t v_size;
2733 :
2734 5667 : ret = sgetbs(f, v_path, sizeof(v_path));
2735 5667 : if (ret < 0) {
2736 : /* LCOV_EXCL_START */
2737 : decoding_error(path, f);
2738 : os_abort();
2739 : /* LCOV_EXCL_STOP */
2740 : }
2741 :
2742 5666 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2743 5666 : if (ret < 0) {
2744 : /* LCOV_EXCL_START */
2745 : decoding_error(path, f);
2746 : os_abort();
2747 : /* LCOV_EXCL_STOP */
2748 : }
2749 :
2750 5666 : ret = sgetb64(f, &v_size);
2751 5666 : if (ret < 0) {
2752 : /* LCOV_EXCL_START */
2753 : decoding_error(path, f);
2754 : os_abort();
2755 : /* LCOV_EXCL_STOP */
2756 : }
2757 :
2758 : /* if we use this parity entry */
2759 5666 : if (v_level < state->level) {
2760 : /* if this split was removed from the configuration */
2761 4630 : if (s >= state->parity[v_level].split_mac) {
2762 : /* if the file is used, we really need it */
2763 0 : if (v_size != 0) {
2764 : /* LCOV_EXCL_START */
2765 : decoding_error(path, f);
2766 : log_fatal("Parity '%s' misses used file '%u'!\n", lev_config_name(v_level), s);
2767 : log_fatal("If you have removed it from the configuration file, please restore it\n");
2768 : exit(EXIT_FAILURE);
2769 : /* LCOV_EXCL_STOP */
2770 : }
2771 :
2772 : /* otherwise we can drop it */
2773 0 : log_fatal("WARNING! Dropping from '%s' unused split '%u'\n", lev_config_name(v_level), s);
2774 : } else {
2775 : /* we copy the path only if without configuration file */
2776 4630 : if (state->no_conf)
2777 24 : pathcpy(state->parity[v_level].split_map[s].path, sizeof(state->parity[v_level].split_map[s].path), v_path);
2778 :
2779 : /* set the split info */
2780 4630 : pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
2781 4630 : state->parity[v_level].split_map[s].size = v_size;
2782 :
2783 : /* log the info read from the content file */
2784 9260 : log_tag("content:%s:%u:%s:%s:%" PRIi64 "\n", lev_config_name(v_level), s,
2785 4630 : state->parity[v_level].split_map[s].path,
2786 4630 : state->parity[v_level].split_map[s].uuid,
2787 : state->parity[v_level].split_map[s].size);
2788 : }
2789 : }
2790 : }
2791 253 : } else if (c == 'N') {
2792 : uint32_t crc_stored;
2793 : uint32_t crc_computed;
2794 :
2795 : /* get the crc before reading it from the file */
2796 253 : crc_computed = scrc(f);
2797 :
2798 253 : ret = sgetble32(f, &crc_stored);
2799 253 : if (ret < 0) {
2800 : /* LCOV_EXCL_START */
2801 : /* here don't call decoding_error() because it's too late to get the crc */
2802 : log_fatal("Error reading the CRC in '%s' at offset %" PRIi64 "\n", path, stell(f));
2803 : log_fatal("This content file is damaged! Use an alternate copy.\n");
2804 : exit(EXIT_FAILURE);
2805 : /* LCOV_EXCL_STOP */
2806 : }
2807 :
2808 253 : if (crc_stored != crc_computed) {
2809 : /* LCOV_EXCL_START */
2810 : /* here don't call decoding_error() because it's too late to get the crc */
2811 : log_fatal("Mismatching CRC in '%s'\n", path);
2812 : log_fatal("This content file is damaged! Use an alternate copy.\n");
2813 : exit(EXIT_FAILURE);
2814 : /* LCOV_EXCL_STOP */
2815 : }
2816 :
2817 253 : crc_checked = 1;
2818 : } else {
2819 : /* LCOV_EXCL_START */
2820 : decoding_error(path, f);
2821 : log_fatal("Invalid command '%c'!\n", (char)c);
2822 : os_abort();
2823 : /* LCOV_EXCL_STOP */
2824 : }
2825 3735359 : }
2826 :
2827 253 : tommy_array_done(&disk_mapping);
2828 :
2829 253 : if (serror(f)) {
2830 : /* LCOV_EXCL_START */
2831 : log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
2832 : exit(EXIT_FAILURE);
2833 : /* LCOV_EXCL_STOP */
2834 : }
2835 :
2836 253 : if (!crc_checked) {
2837 : /* LCOV_EXCL_START */
2838 : log_fatal("Finished reading '%s' without finding the CRC\n", path);
2839 : log_fatal("This content file is truncated or damaged! Use an alternate copy.\n");
2840 : exit(EXIT_FAILURE);
2841 : /* LCOV_EXCL_STOP */
2842 : }
2843 :
2844 : /* check the file-system on all disks */
2845 253 : state_fscheck(state, "after read");
2846 :
2847 : /* check that the stored parity size matches the loaded state */
2848 253 : if (blockmax != parity_allocated_size(state)) {
2849 : /* LCOV_EXCL_START */
2850 : log_fatal("Internal inconsistency in parity size %u/%u in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f));
2851 : if (state->opt.skip_content_check) {
2852 : log_fatal("Overriding.\n");
2853 : blockmax = parity_allocated_size(state);
2854 : } else {
2855 : exit(EXIT_FAILURE);
2856 : }
2857 : /* LCOV_EXCL_STOP */
2858 : }
2859 :
2860 253 : msg_verbose("%8u files\n", count_file);
2861 253 : msg_verbose("%8u hardlinks\n", count_hardlink);
2862 253 : msg_verbose("%8u symlinks\n", count_symlink);
2863 253 : msg_verbose("%8u empty dirs\n", count_dir);
2864 253 : }
2865 :
2866 : struct state_write_thread_context {
2867 : struct snapraid_state* state;
2868 : #if HAVE_MT_WRITE
2869 : pthread_t thread;
2870 : #endif
2871 : /* input */
2872 : block_off_t blockmax;
2873 : time_t info_oldest;
2874 : int info_has_rehash;
2875 : STREAM* f;
2876 : /* output */
2877 : uint32_t crc;
2878 : unsigned count_file;
2879 : unsigned count_hardlink;
2880 : unsigned count_symlink;
2881 : unsigned count_dir;
2882 : };
2883 :
2884 151 : static void* state_write_thread(void* arg)
2885 : {
2886 151 : struct state_write_thread_context* context = arg;
2887 151 : struct snapraid_state* state = context->state;
2888 151 : block_off_t blockmax = context->blockmax;
2889 151 : time_t info_oldest = context->info_oldest;
2890 151 : int info_has_rehash = context->info_has_rehash;
2891 151 : STREAM* f = context->f;
2892 : uint32_t crc;
2893 : unsigned count_file;
2894 : unsigned count_hardlink;
2895 : unsigned count_symlink;
2896 : unsigned count_dir;
2897 : tommy_node* i;
2898 : block_off_t idx;
2899 : block_off_t begin;
2900 : unsigned l, s;
2901 : int version;
2902 :
2903 151 : count_file = 0;
2904 151 : count_hardlink = 0;
2905 151 : count_symlink = 0;
2906 151 : count_dir = 0;
2907 :
2908 : /* check what version to use */
2909 151 : version = 2;
2910 1012 : for (l = 0; l < state->level; ++l) {
2911 861 : if (state->parity[l].split_mac > 1)
2912 861 : version = 3;
2913 : }
2914 151 : if (BLOCK_HASH_SIZE != 16)
2915 0 : version = 3;
2916 :
2917 : /* write header */
2918 151 : if (version == 3)
2919 151 : swrite("SNAPCNT3\n\3\0\0", 12, f);
2920 : else
2921 0 : swrite("SNAPCNT2\n\3\0\0", 12, f);
2922 :
2923 : /* write block size and block max */
2924 151 : sputc('z', f);
2925 151 : sputb32(state->block_size, f);
2926 151 : sputc('x', f);
2927 151 : sputb32(blockmax, f);
2928 :
2929 : /* hash size */
2930 151 : if (version == 3) {
2931 151 : sputc('y', f);
2932 151 : sputb32(BLOCK_HASH_SIZE, f);
2933 : }
2934 :
2935 151 : if (serror(f)) {
2936 : /* LCOV_EXCL_START */
2937 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2938 : return context;
2939 : /* LCOV_EXCL_STOP */
2940 : }
2941 :
2942 151 : sputc('c', f);
2943 151 : if (state->hash == HASH_MURMUR3) {
2944 122 : sputc('u', f);
2945 29 : } else if (state->hash == HASH_SPOOKY2) {
2946 29 : sputc('k', f);
2947 : } else {
2948 : /* LCOV_EXCL_START */
2949 : log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f));
2950 : return context;
2951 : /* LCOV_EXCL_STOP */
2952 : }
2953 151 : swrite(state->hashseed, HASH_MAX, f);
2954 151 : if (serror(f)) {
2955 : /* LCOV_EXCL_START */
2956 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2957 : return context;
2958 : /* LCOV_EXCL_STOP */
2959 : }
2960 :
2961 : /* previous hash only present */
2962 151 : if (state->prevhash != HASH_UNDEFINED) {
2963 : /* if at least one rehash tag found, we have to save the previous hash */
2964 3 : if (info_has_rehash) {
2965 2 : sputc('C', f);
2966 2 : if (state->prevhash == HASH_MURMUR3) {
2967 2 : sputc('u', f);
2968 0 : } else if (state->prevhash == HASH_SPOOKY2) {
2969 0 : sputc('k', f);
2970 : } else {
2971 : /* LCOV_EXCL_START */
2972 : log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
2973 : return context;
2974 : /* LCOV_EXCL_STOP */
2975 : }
2976 2 : swrite(state->prevhashseed, HASH_MAX, f);
2977 2 : if (serror(f)) {
2978 : /* LCOV_EXCL_START */
2979 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2980 : return context;
2981 : /* LCOV_EXCL_STOP */
2982 : }
2983 : }
2984 : }
2985 :
2986 : /* for each map */
2987 1057 : for (i = state->maplist; i != 0; i = i->next) {
2988 906 : struct snapraid_map* map = i->data;
2989 : struct snapraid_disk* disk;
2990 :
2991 : /* find the disk for this mapping */
2992 906 : disk = find_disk_by_name(state, map->name);
2993 906 : if (!disk) {
2994 : /* LCOV_EXCL_START */
2995 : log_fatal("Internal inconsistency for unmapped disk '%s'\n", map->name);
2996 : return context;
2997 : /* LCOV_EXCL_STOP */
2998 : }
2999 :
3000 : /* save the mapping only if disk is mapped */
3001 906 : if (disk->mapping_idx != -1) {
3002 814 : sputc('M', f);
3003 814 : sputbs(map->name, f);
3004 814 : sputb32(map->position, f);
3005 814 : sputb32(map->total_blocks, f);
3006 814 : sputb32(map->free_blocks, f);
3007 814 : sputbs(map->uuid, f);
3008 814 : if (serror(f)) {
3009 : /* LCOV_EXCL_START */
3010 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3011 : return context;
3012 : /* LCOV_EXCL_STOP */
3013 : }
3014 : }
3015 : }
3016 :
3017 : /* for each parity */
3018 1012 : for (l = 0; l < state->level; ++l) {
3019 861 : if (version == 3) {
3020 861 : sputc('Q', f);
3021 861 : sputb32(l, f);
3022 861 : sputb32(state->parity[l].total_blocks, f);
3023 861 : sputb32(state->parity[l].free_blocks, f);
3024 861 : sputb32(state->parity[l].split_mac, f);
3025 4305 : for (s = 0; s < state->parity[l].split_mac; ++s) {
3026 3444 : sputbs(state->parity[l].split_map[s].path, f);
3027 3444 : sputbs(state->parity[l].split_map[s].uuid, f);
3028 3444 : sputb64(state->parity[l].split_map[s].size, f);
3029 : }
3030 : } else {
3031 0 : sputc('P', f);
3032 0 : sputb32(l, f);
3033 0 : sputb32(state->parity[l].total_blocks, f);
3034 0 : sputb32(state->parity[l].free_blocks, f);
3035 0 : sputbs(state->parity[l].split_map[0].uuid, f);
3036 : }
3037 861 : if (serror(f)) {
3038 : /* LCOV_EXCL_START */
3039 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3040 : return context;
3041 : /* LCOV_EXCL_STOP */
3042 : }
3043 : }
3044 :
3045 : /* for each disk */
3046 1057 : for (i = state->disklist; i != 0; i = i->next) {
3047 : tommy_node* j;
3048 906 : struct snapraid_disk* disk = i->data;
3049 :
3050 : /* if the disk is not mapped, skip it */
3051 906 : if (disk->mapping_idx < 0)
3052 92 : continue;
3053 :
3054 : /* for each file */
3055 1771698 : for (j = disk->filelist; j != 0; j = j->next) {
3056 1770884 : struct snapraid_file* file = j->data;
3057 : uint64_t size;
3058 : uint64_t mtime_sec;
3059 : int32_t mtime_nsec;
3060 : uint64_t inode;
3061 :
3062 1770884 : size = file->size;
3063 1770884 : mtime_sec = file->mtime_sec;
3064 1770884 : mtime_nsec = file->mtime_nsec;
3065 1770884 : inode = file->inode;
3066 :
3067 1770884 : sputc('f', f);
3068 1770884 : sputb32(disk->mapping_idx, f);
3069 1770884 : sputb64(size, f);
3070 1770884 : sputb64(mtime_sec, f);
3071 : /* encode STAT_NSEC_INVALID as 0 */
3072 1770884 : if (mtime_nsec == STAT_NSEC_INVALID)
3073 0 : sputb32(0, f);
3074 : else
3075 1770884 : sputb32(mtime_nsec + 1, f);
3076 1770884 : sputb64(inode, f);
3077 1770884 : sputbs(file->sub, f);
3078 1770884 : if (serror(f)) {
3079 : /* LCOV_EXCL_START */
3080 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3081 : return context;
3082 : /* LCOV_EXCL_STOP */
3083 : }
3084 :
3085 : /* for all the blocks of the file */
3086 1770884 : begin = 0;
3087 5323011 : while (begin < file->blockmax) {
3088 1781243 : unsigned v_state = block_state_get(fs_file2block_get(file, begin));
3089 1781243 : block_off_t v_pos = fs_file2par_get(disk, file, begin);
3090 : uint32_t v_count;
3091 :
3092 : block_off_t end;
3093 :
3094 : /* find the end of run of blocks */
3095 1781243 : end = begin + 1;
3096 6146383 : while (end < file->blockmax) {
3097 2596728 : if (v_state != block_state_get(fs_file2block_get(file, end)))
3098 12 : break;
3099 2596716 : if (v_pos + (end - begin) != fs_file2par_get(disk, file, end))
3100 12819 : break;
3101 2583897 : ++end;
3102 : }
3103 :
3104 1781243 : switch (v_state) {
3105 : case BLOCK_STATE_BLK :
3106 1678566 : sputc('b', f);
3107 1678566 : break;
3108 : case BLOCK_STATE_CHG :
3109 67759 : sputc('g', f);
3110 67759 : break;
3111 : case BLOCK_STATE_REP :
3112 34918 : sputc('p', f);
3113 34918 : break;
3114 : default :
3115 : /* LCOV_EXCL_START */
3116 : log_fatal("Internal inconsistency in state for block %u state %u\n", v_pos, v_state);
3117 : return context;
3118 : /* LCOV_EXCL_STOP */
3119 : }
3120 :
3121 1781243 : sputb32(v_pos, f);
3122 :
3123 1781243 : v_count = end - begin;
3124 1781243 : sputb32(v_count, f);
3125 :
3126 : /* write hashes */
3127 6146383 : for (idx = begin; idx < end; ++idx) {
3128 4365140 : struct snapraid_block* block = fs_file2block_get(file, idx);
3129 :
3130 4365140 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3131 : }
3132 :
3133 1781243 : if (serror(f)) {
3134 : /* LCOV_EXCL_START */
3135 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3136 : return context;
3137 : /* LCOV_EXCL_STOP */
3138 : }
3139 :
3140 : /* next begin position */
3141 1781243 : begin = end;
3142 : }
3143 :
3144 1770884 : ++count_file;
3145 : }
3146 :
3147 : /* for each link */
3148 58887 : for (j = disk->linklist; j != 0; j = j->next) {
3149 58073 : struct snapraid_link* slink = j->data;
3150 :
3151 58073 : switch (link_flag_get(slink, FILE_IS_LINK_MASK)) {
3152 : case FILE_IS_HARDLINK :
3153 321 : sputc('a', f);
3154 321 : ++count_hardlink;
3155 321 : break;
3156 : case FILE_IS_SYMLINK :
3157 57752 : sputc('s', f);
3158 57752 : ++count_symlink;
3159 57752 : break;
3160 : }
3161 :
3162 58073 : sputb32(disk->mapping_idx, f);
3163 58073 : sputbs(slink->sub, f);
3164 58073 : sputbs(slink->linkto, f);
3165 58073 : if (serror(f)) {
3166 : /* LCOV_EXCL_START */
3167 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3168 : return context;
3169 : /* LCOV_EXCL_STOP */
3170 : }
3171 : }
3172 :
3173 : /* for each dir */
3174 1241 : for (j = disk->dirlist; j != 0; j = j->next) {
3175 427 : struct snapraid_dir* dir = j->data;
3176 :
3177 427 : sputc('r', f);
3178 427 : sputb32(disk->mapping_idx, f);
3179 427 : sputbs(dir->sub, f);
3180 427 : if (serror(f)) {
3181 : /* LCOV_EXCL_START */
3182 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3183 : return context;
3184 : /* LCOV_EXCL_STOP */
3185 : }
3186 :
3187 427 : ++count_dir;
3188 : }
3189 :
3190 : /* deleted blocks of the disk */
3191 814 : sputc('h', f);
3192 814 : sputb32(disk->mapping_idx, f);
3193 814 : if (serror(f)) {
3194 : /* LCOV_EXCL_START */
3195 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3196 : return context;
3197 : /* LCOV_EXCL_STOP */
3198 : }
3199 814 : begin = 0;
3200 2516 : while (begin < blockmax) {
3201 : int is_deleted;
3202 : block_off_t end;
3203 :
3204 888 : is_deleted = fs_is_block_deleted(disk, begin);
3205 :
3206 : /* find the end of run of blocks */
3207 888 : end = begin + 1;
3208 4530435 : while (end < blockmax
3209 4528733 : && is_deleted == fs_is_block_deleted(disk, end)
3210 : ) {
3211 4528659 : ++end;
3212 : }
3213 :
3214 888 : sputb32(end - begin, f);
3215 :
3216 888 : if (is_deleted) {
3217 : /* write the run of deleted blocks with hash */
3218 47 : sputc('o', f);
3219 :
3220 : /* write all the hash */
3221 15961 : while (begin < end) {
3222 15867 : struct snapraid_block* block = fs_par2block_get(disk, begin);
3223 :
3224 15867 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3225 :
3226 15867 : ++begin;
3227 : }
3228 : } else {
3229 : /* write the run of blocks without hash */
3230 : /* they can be either used or empty blocks */
3231 841 : sputc('O', f);
3232 :
3233 : /* next begin position */
3234 841 : begin = end;
3235 : }
3236 :
3237 888 : if (serror(f)) {
3238 : /* LCOV_EXCL_START */
3239 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3240 : return context;
3241 : /* LCOV_EXCL_STOP */
3242 : }
3243 : }
3244 : }
3245 :
3246 : /* write the info for each block */
3247 151 : sputc('i', f);
3248 151 : sputb32(info_oldest, f);
3249 151 : begin = 0;
3250 19719 : while (begin < blockmax) {
3251 : snapraid_info info;
3252 : block_off_t end;
3253 : time_t t;
3254 : unsigned flag;
3255 :
3256 19417 : info = info_get(&state->infoarr, begin);
3257 :
3258 : /* find the end of run of blocks */
3259 19417 : end = begin + 1;
3260 775516 : while (end < blockmax
3261 755951 : && info == info_get(&state->infoarr, end)
3262 : ) {
3263 736682 : ++end;
3264 : }
3265 :
3266 19417 : sputb32(end - begin, f);
3267 :
3268 : /* if there is info */
3269 19417 : if (info) {
3270 : /* other flags */
3271 19394 : flag = 1; /* info is present */
3272 19394 : if (info_get_bad(info))
3273 1348 : flag |= 2;
3274 19394 : if (info_get_rehash(info))
3275 5212 : flag |= 4;
3276 19394 : if (info_get_justsynced(info))
3277 12059 : flag |= 8;
3278 19394 : sputb32(flag, f);
3279 :
3280 19394 : t = info_get_time(info) - info_oldest;
3281 19394 : sputb32(t, f);
3282 : } else {
3283 : /* write a special 0 flag to mark missing info */
3284 23 : sputb32(0, f);
3285 : }
3286 :
3287 19417 : if (serror(f)) {
3288 : /* LCOV_EXCL_START */
3289 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3290 : return context;
3291 : /* LCOV_EXCL_STOP */
3292 : }
3293 :
3294 : /* next begin position */
3295 19417 : begin = end;
3296 : }
3297 :
3298 151 : sputc('N', f);
3299 :
3300 : /* flush data written to the disk */
3301 151 : if (sflush(f)) {
3302 : /* LCOV_EXCL_START */
3303 : log_fatal("Error writing the content file '%s' (in flush before crc). %s.\n", serrorfile(f), strerror(errno));
3304 : return context;
3305 : /* LCOV_EXCL_STOP */
3306 : }
3307 :
3308 : /* get the file crc */
3309 151 : crc = scrc(f);
3310 :
3311 : /* compare the crc of the data written to file */
3312 : /* with the one of the data written to the stream */
3313 151 : if (crc != scrc_stream(f)) {
3314 : /* LCOV_EXCL_START */
3315 : log_fatal("CRC mismatch writing the content stream.\n");
3316 : log_fatal("DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
3317 : log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3318 : return context;
3319 : /* LCOV_EXCL_STOP */
3320 : }
3321 :
3322 151 : sputble32(crc, f);
3323 151 : if (serror(f)) {
3324 : /* LCOV_EXCL_START */
3325 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3326 : return context;
3327 : /* LCOV_EXCL_STOP */
3328 : }
3329 :
3330 : /* set output variables */
3331 151 : context->crc = crc;
3332 151 : context->count_file = count_file;
3333 151 : context->count_hardlink = count_hardlink;
3334 151 : context->count_symlink = count_symlink;
3335 151 : context->count_dir = count_dir;
3336 :
3337 151 : return 0;
3338 : }
3339 :
3340 151 : static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
3341 : {
3342 : #if HAVE_MT_WRITE
3343 : int fail;
3344 : int first;
3345 : #else
3346 : STREAM* f;
3347 : unsigned count_content;
3348 : unsigned k;
3349 : struct state_write_thread_context* context;
3350 : void* retval;
3351 : #endif
3352 : tommy_node* i;
3353 : block_off_t blockmax;
3354 : time_t info_oldest;
3355 : int info_has_rehash;
3356 : int mapping_idx;
3357 : block_off_t idx;
3358 : uint32_t crc;
3359 : unsigned count_file;
3360 : unsigned count_hardlink;
3361 : unsigned count_symlink;
3362 : unsigned count_dir;
3363 :
3364 : /* blocks of all array */
3365 151 : blockmax = parity_allocated_size(state);
3366 :
3367 : /* check the file-system on all disks */
3368 151 : state_fscheck(state, "before write");
3369 :
3370 : /* clear the info for unused blocks */
3371 : /* and get some other info */
3372 151 : info_oldest = 0; /* oldest time in info */
3373 151 : info_has_rehash = 0; /* if there is a rehash info */
3374 756250 : for (idx = 0; idx < blockmax; ++idx) {
3375 : /* if the position is used */
3376 756099 : if (fs_position_is_required(state, idx)) {
3377 756099 : snapraid_info info = info_get(&state->infoarr, idx);
3378 :
3379 : /* only if there is some info to store */
3380 756099 : if (info) {
3381 711614 : time_t info_time = info_get_time(info);
3382 :
3383 711614 : if (!info_oldest || info_time < info_oldest)
3384 151 : info_oldest = info_time;
3385 :
3386 711614 : if (info_get_rehash(info))
3387 13822 : info_has_rehash = 1;
3388 : }
3389 : } else {
3390 : /* clear any previous info */
3391 0 : info_set(&state->infoarr, idx, 0);
3392 :
3393 : /* and clear any deleted blocks */
3394 0 : fs_position_clear_deleted(state, idx);
3395 : }
3396 : }
3397 :
3398 : /* map disks */
3399 151 : mapping_idx = 0;
3400 1057 : for (i = state->maplist; i != 0; i = i->next) {
3401 906 : struct snapraid_map* map = i->data;
3402 : struct snapraid_disk* disk;
3403 :
3404 : /* find the disk for this mapping */
3405 906 : disk = find_disk_by_name(state, map->name);
3406 906 : if (!disk) {
3407 : /* LCOV_EXCL_START */
3408 : log_fatal("Internal inconsistency for unmapped disk '%s'\n", map->name);
3409 : os_abort();
3410 : /* LCOV_EXCL_STOP */
3411 : }
3412 :
3413 : /* save the mapping only for not empty disks */
3414 906 : if (!fs_is_empty(disk, blockmax)) {
3415 : /* assign the mapping index used to identify disks */
3416 814 : disk->mapping_idx = mapping_idx;
3417 814 : ++mapping_idx;
3418 : } else {
3419 : /* mark the disk as without mapping */
3420 92 : disk->mapping_idx = -1;
3421 : }
3422 : }
3423 :
3424 : #if HAVE_MT_WRITE
3425 : /* start all writing threads */
3426 : i = tommy_list_head(&state->contentlist);
3427 : while (i) {
3428 : struct snapraid_content* content = i->data;
3429 : struct state_write_thread_context* context;
3430 : char tmp[PATH_MAX];
3431 : STREAM* f;
3432 :
3433 : msg_progress("Saving state to %s...\n", content->content);
3434 :
3435 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3436 : f = sopen_write(tmp);
3437 : if (f == 0) {
3438 : /* LCOV_EXCL_START */
3439 : log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
3440 : exit(EXIT_FAILURE);
3441 : /* LCOV_EXCL_STOP */
3442 : }
3443 :
3444 : /* allocate the thread context */
3445 : context = malloc_nofail(sizeof(struct state_write_thread_context));
3446 : content->context = context;
3447 :
3448 : /* initialize */
3449 : context->state = state;
3450 : context->blockmax = blockmax;
3451 : context->info_oldest = info_oldest;
3452 : context->info_has_rehash = info_has_rehash;
3453 : context->f = f;
3454 :
3455 : thread_create(&context->thread, 0, state_write_thread, context);
3456 :
3457 : i = i->next;
3458 : }
3459 :
3460 : /* join all thread */
3461 : fail = 0;
3462 : first = 1;
3463 : crc = 0;
3464 : count_file = 0;
3465 : count_hardlink = 0;
3466 : count_symlink = 0;
3467 : count_dir = 0;
3468 : i = tommy_list_head(&state->contentlist);
3469 : while (i) {
3470 : struct snapraid_content* content = i->data;
3471 : struct state_write_thread_context* context = content->context;
3472 : void* retval;
3473 :
3474 : thread_join(context->thread, &retval);
3475 :
3476 : if (retval) {
3477 : /* LCOV_EXCL_START */
3478 : fail = 1;
3479 : /* LCOV_EXCL_STOP */
3480 : } else {
3481 : STREAM* f = context->f;
3482 :
3483 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3484 : /* than even in a system crash event we have one valid copy of the file. */
3485 : if (sflush(f) != 0) {
3486 : /* LCOV_EXCL_START */
3487 : log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3488 : exit(EXIT_FAILURE);
3489 : /* LCOV_EXCL_STOP */
3490 : }
3491 :
3492 : #if HAVE_FSYNC
3493 : if (ssync(f) != 0) {
3494 : /* LCOV_EXCL_START */
3495 : log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3496 : exit(EXIT_FAILURE);
3497 : /* LCOV_EXCL_STOP */
3498 : }
3499 : #endif
3500 :
3501 : if (sclose(f) != 0) {
3502 : /* LCOV_EXCL_START */
3503 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3504 : exit(EXIT_FAILURE);
3505 : /* LCOV_EXCL_STOP */
3506 : }
3507 :
3508 : if (first) {
3509 : first = 0;
3510 : crc = context->crc;
3511 : count_file = context->count_file;
3512 : count_hardlink = context->count_hardlink;
3513 : count_symlink = context->count_symlink;
3514 : count_dir = context->count_dir;
3515 : } else {
3516 : if (crc != context->crc) {
3517 : /* LCOV_EXCL_START */
3518 : log_fatal("Different CRCs writing content streams.\n");
3519 : log_fatal("DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
3520 : log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3521 : exit(EXIT_FAILURE);
3522 : /* LCOV_EXCL_STOP */
3523 : }
3524 : }
3525 : }
3526 :
3527 : free(context);
3528 :
3529 : i = i->next;
3530 : }
3531 :
3532 : /* abort on failure */
3533 : if (fail) {
3534 : /* LCOV_EXCL_START */
3535 : exit(EXIT_FAILURE);
3536 : /* LCOV_EXCL_STOP */
3537 : }
3538 : #else
3539 : /* count the content files */
3540 151 : count_content = 0;
3541 151 : i = tommy_list_head(&state->contentlist);
3542 1314 : while (i) {
3543 1012 : struct snapraid_content* content = i->data;
3544 1012 : msg_progress("Saving state to %s...\n", content->content);
3545 1012 : ++count_content;
3546 1012 : i = i->next;
3547 : }
3548 :
3549 : /* open all the content files */
3550 151 : f = sopen_multi_write(count_content);
3551 151 : if (!f) {
3552 : /* LCOV_EXCL_START */
3553 : log_fatal("Error opening the content files.\n");
3554 : exit(EXIT_FAILURE);
3555 : /* LCOV_EXCL_STOP */
3556 : }
3557 :
3558 151 : k = 0;
3559 151 : i = tommy_list_head(&state->contentlist);
3560 1314 : while (i) {
3561 1012 : struct snapraid_content* content = i->data;
3562 : char tmp[PATH_MAX];
3563 1012 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3564 1012 : if (sopen_multi_file(f, k, tmp) != 0) {
3565 : /* LCOV_EXCL_START */
3566 : log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
3567 : exit(EXIT_FAILURE);
3568 : /* LCOV_EXCL_STOP */
3569 : }
3570 1012 : ++k;
3571 1012 : i = i->next;
3572 : }
3573 :
3574 : /* allocate the thread context */
3575 151 : context = malloc_nofail(sizeof(struct state_write_thread_context));
3576 :
3577 : /* initialize */
3578 151 : context->state = state;
3579 151 : context->blockmax = blockmax;
3580 151 : context->info_oldest = info_oldest;
3581 151 : context->info_has_rehash = info_has_rehash;
3582 151 : context->f = f;
3583 :
3584 151 : retval = state_write_thread(context);
3585 :
3586 : /* abort on failure */
3587 151 : if (retval) {
3588 : /* LCOV_EXCL_START */
3589 : exit(EXIT_FAILURE);
3590 : /* LCOV_EXCL_STOP */
3591 : }
3592 :
3593 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3594 : /* than even in a system crash event we have one valid copy of the file. */
3595 151 : if (sflush(f) != 0) {
3596 : /* LCOV_EXCL_START */
3597 : log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3598 : exit(EXIT_FAILURE);
3599 : /* LCOV_EXCL_STOP */
3600 : }
3601 :
3602 : #if HAVE_FSYNC
3603 151 : if (ssync(f) != 0) {
3604 : /* LCOV_EXCL_START */
3605 : log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3606 : exit(EXIT_FAILURE);
3607 : /* LCOV_EXCL_STOP */
3608 : }
3609 : #endif
3610 :
3611 151 : if (sclose(f) != 0) {
3612 : /* LCOV_EXCL_START */
3613 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3614 : exit(EXIT_FAILURE);
3615 : /* LCOV_EXCL_STOP */
3616 : }
3617 :
3618 151 : crc = context->crc;
3619 151 : count_file = context->count_file;
3620 151 : count_hardlink = context->count_hardlink;
3621 151 : count_symlink = context->count_symlink;
3622 151 : count_dir = context->count_dir;
3623 :
3624 151 : free(context);
3625 : #endif
3626 :
3627 151 : msg_verbose("%8u files\n", count_file);
3628 151 : msg_verbose("%8u hardlinks\n", count_hardlink);
3629 151 : msg_verbose("%8u symlinks\n", count_symlink);
3630 151 : msg_verbose("%8u empty dirs\n", count_dir);
3631 :
3632 151 : *out_crc = crc;
3633 151 : }
3634 :
3635 261 : void state_read(struct snapraid_state* state)
3636 : {
3637 : STREAM* f;
3638 : char path[PATH_MAX];
3639 : struct stat st;
3640 : tommy_node* node;
3641 : int ret;
3642 : int c;
3643 :
3644 : /* iterate over all the available content files and load the first one present */
3645 261 : f = 0;
3646 261 : node = tommy_list_head(&state->contentlist);
3647 572 : while (node) {
3648 304 : struct snapraid_content* content = node->data;
3649 304 : pathcpy(path, sizeof(path), content->content);
3650 :
3651 304 : if (!state->no_conf) {
3652 303 : log_tag("content:%s\n", path);
3653 303 : log_flush();
3654 : }
3655 304 : msg_progress("Loading state from %s...\n", path);
3656 :
3657 304 : f = sopen_read(path);
3658 304 : if (f != 0) {
3659 : /* if opened stop the search */
3660 254 : break;
3661 : } else {
3662 : /* if it's real error of an existing file, abort */
3663 50 : if (errno != ENOENT) {
3664 : /* LCOV_EXCL_START */
3665 : log_fatal("Error opening the content file '%s'. %s.\n", path, strerror(errno));
3666 : exit(EXIT_FAILURE);
3667 : /* LCOV_EXCL_STOP */
3668 : }
3669 :
3670 : /* otherwise continue */
3671 50 : if (node->next) {
3672 43 : log_fatal("WARNING! Content file '%s' not found, trying with another copy...\n", path);
3673 :
3674 : /* ensure to rewrite all the content files */
3675 43 : state->need_write = 1;
3676 : }
3677 : }
3678 :
3679 : /* next content file */
3680 50 : node = node->next;
3681 : }
3682 :
3683 : /* if not found, assume empty */
3684 261 : if (!f) {
3685 7 : log_fatal("No content file found. Assuming empty.\n");
3686 :
3687 : /* create the initial mapping */
3688 7 : state_map(state);
3689 267 : return;
3690 : }
3691 :
3692 : /* get the stat of the content file */
3693 254 : ret = fstat(shandle(f), &st);
3694 254 : if (ret != 0) {
3695 : /* LCOV_EXCL_START */
3696 : log_fatal("Error stating the content file '%s'. %s.\n", path, strerror(errno));
3697 : exit(EXIT_FAILURE);
3698 : /* LCOV_EXCL_STOP */
3699 : }
3700 :
3701 : /* go further to check other content files */
3702 1955 : while (node) {
3703 : char other_path[PATH_MAX];
3704 : struct stat other_st;
3705 1447 : struct snapraid_content* content = node->data;
3706 1447 : pathcpy(other_path, sizeof(other_path), content->content);
3707 :
3708 1447 : ret = stat(other_path, &other_st);
3709 1447 : if (ret != 0) {
3710 : /* allow missing content files, but not any other kind of error */
3711 1 : if (errno != ENOENT) {
3712 : /* LCOV_EXCL_START */
3713 : log_fatal("Error stating the content file '%s'. %s.\n", other_path, strerror(errno));
3714 : exit(EXIT_FAILURE);
3715 : /* LCOV_EXCL_STOP */
3716 : }
3717 :
3718 : /* ensure to rewrite all the content files */
3719 1 : state->need_write = 1;
3720 : } else {
3721 : /* if the size is different */
3722 1446 : if (other_st.st_size != st.st_size) {
3723 39 : log_fatal("WARNING! Content files '%s' and '%s' have a different size!\n", path, other_path);
3724 39 : log_fatal("Likely one of the two is broken!\n");
3725 :
3726 : /* ensure to rewrite all the content files */
3727 39 : state->need_write = 1;
3728 : }
3729 : }
3730 :
3731 : /* next content file */
3732 1447 : node = node->next;
3733 : }
3734 :
3735 : /* start with a undefined default. */
3736 : /* it's for compatibility with version 1.0 where MD5 was implicit. */
3737 254 : state->hash = HASH_UNDEFINED;
3738 :
3739 : /* start with a zero seed, it was the default in old versions */
3740 254 : memset(state->hashseed, 0, HASH_MAX);
3741 :
3742 : /* previous hash, start with an undefined value */
3743 254 : state->prevhash = HASH_UNDEFINED;
3744 :
3745 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
3746 :
3747 : /* get the first char to detect the file type */
3748 254 : c = sgetc(f);
3749 254 : sungetc(c, f);
3750 :
3751 : /* guess the file type from the first char */
3752 254 : if (c == 'S') {
3753 254 : state_read_content(state, path, f);
3754 : } else {
3755 : /* LCOV_EXCL_START */
3756 : log_fatal("From SnapRAID v9.0 the text content file is not supported anymore.\n");
3757 : log_fatal("You have first to upgrade to SnapRAID v8.1 to convert it to binary format.\n");
3758 : exit(EXIT_FAILURE);
3759 : /* LCOV_EXCL_STOP */
3760 : }
3761 :
3762 253 : sclose(f);
3763 :
3764 253 : if (state->hash == HASH_UNDEFINED) {
3765 : /* LCOV_EXCL_START */
3766 : log_fatal("The checksum to use is not specified.\n");
3767 : log_fatal("This happens because you are likely upgrading from SnapRAID 1.0.\n");
3768 : log_fatal("To use a new SnapRAID you must restart from scratch,\n");
3769 : log_fatal("deleting all the content and parity files.\n");
3770 : exit(EXIT_FAILURE);
3771 : /* LCOV_EXCL_STOP */
3772 : }
3773 :
3774 : /* update the mapping */
3775 253 : state_map(state);
3776 :
3777 253 : state_content_check(state, path);
3778 :
3779 : /* mark that we read the content file, and it passed all the checks */
3780 253 : state->checked_read = 1;
3781 : }
3782 :
3783 : struct state_verify_thread_context {
3784 : struct snapraid_state* state;
3785 : struct snapraid_content* content;
3786 : #if HAVE_MT_VERIFY
3787 : pthread_t thread;
3788 : #else
3789 : void* retval;
3790 : #endif
3791 : /* input */
3792 : uint32_t crc;
3793 : STREAM* f;
3794 : };
3795 :
3796 1009 : static void* state_verify_thread(void* arg)
3797 : {
3798 1009 : struct state_verify_thread_context* context = arg;
3799 1009 : STREAM* f = context->f;
3800 : unsigned char buf[4];
3801 : uint32_t crc_stored;
3802 : uint32_t crc_computed;
3803 :
3804 1009 : if (sdeplete(f, buf) != 0) {
3805 : /* LCOV_EXCL_START */
3806 : log_fatal("Error flushing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3807 : return context;
3808 : /* LCOV_EXCL_STOP */
3809 : }
3810 :
3811 : /* get the stored crc from the last four bytes */
3812 1012 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
3813 :
3814 1012 : if (crc_stored != context->crc) {
3815 : /* LCOV_EXCL_START */
3816 : log_fatal("DANGER! Wrong stored CRC in '%s'\n", serrorfile(f));
3817 : return context;
3818 : /* LCOV_EXCL_STOP */
3819 : }
3820 :
3821 : /* get the computed crc */
3822 1012 : crc_computed = scrc(f);
3823 :
3824 : /* adjust the stored crc to include itself */
3825 1012 : crc_stored = crc32c(crc_stored, buf, 4);
3826 :
3827 1012 : if (crc_computed != crc_stored) {
3828 : /* LCOV_EXCL_START */
3829 : log_fatal("DANGER! Wrong file CRC in '%s'\n", serrorfile(f));
3830 : return context;
3831 : /* LCOV_EXCL_STOP */
3832 : }
3833 :
3834 1012 : return 0;
3835 : }
3836 :
3837 151 : static void state_verify_content(struct snapraid_state* state, uint32_t crc)
3838 : {
3839 : tommy_node* i;
3840 : int fail;
3841 :
3842 : /* start all reading threads */
3843 151 : i = tommy_list_head(&state->contentlist);
3844 1314 : while (i) {
3845 1012 : struct snapraid_content* content = i->data;
3846 : struct state_verify_thread_context* context;
3847 : char tmp[PATH_MAX];
3848 : STREAM* f;
3849 :
3850 1012 : msg_progress("Verifying %s...\n", content->content);
3851 :
3852 1012 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3853 :
3854 1012 : f = sopen_read(tmp);
3855 1012 : if (f == 0) {
3856 : /* LCOV_EXCL_START */
3857 : log_fatal("Error reopening the content file '%s'. %s.\n", tmp, strerror(errno));
3858 : exit(EXIT_FAILURE);
3859 : /* LCOV_EXCL_STOP */
3860 : }
3861 :
3862 : /* allocate the thread context */
3863 1012 : context = malloc_nofail(sizeof(struct state_verify_thread_context));
3864 1012 : content->context = context;
3865 :
3866 : /* initialize */
3867 1012 : context->state = state;
3868 1012 : context->content = content;
3869 1012 : context->crc = crc;
3870 1012 : context->f = f;
3871 :
3872 : #if HAVE_MT_VERIFY
3873 1012 : thread_create(&context->thread, 0, state_verify_thread, context);
3874 : #else
3875 : context->retval = state_verify_thread(context);
3876 : #endif
3877 :
3878 1012 : i = i->next;
3879 : }
3880 :
3881 : /* join all thread */
3882 151 : fail = 0;
3883 151 : i = tommy_list_head(&state->contentlist);
3884 1314 : while (i) {
3885 1012 : struct snapraid_content* content = i->data;
3886 1012 : struct state_verify_thread_context* context = content->context;
3887 : void* retval;
3888 :
3889 : #if HAVE_MT_VERIFY
3890 1012 : thread_join(context->thread, &retval);
3891 : #else
3892 : retval = context->retval;
3893 : #endif
3894 1012 : if (retval) {
3895 : /* LCOV_EXCL_START */
3896 : fail = 1;
3897 : /* LCOV_EXCL_STOP */
3898 : } else {
3899 1012 : STREAM* f = context->f;
3900 :
3901 1012 : if (sclose(f) != 0) {
3902 : /* LCOV_EXCL_START */
3903 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3904 : exit(EXIT_FAILURE);
3905 : /* LCOV_EXCL_STOP */
3906 : }
3907 : }
3908 :
3909 1012 : free(context);
3910 :
3911 1012 : i = i->next;
3912 : }
3913 :
3914 : /* abort on failure */
3915 151 : if (fail) {
3916 : /* LCOV_EXCL_START */
3917 : exit(EXIT_FAILURE);
3918 : /* LCOV_EXCL_STOP */
3919 : }
3920 151 : }
3921 :
3922 151 : static void state_rename_content(struct snapraid_state* state)
3923 : {
3924 : tommy_node* i;
3925 :
3926 151 : i = tommy_list_head(&state->contentlist);
3927 1314 : while (i) {
3928 1012 : struct snapraid_content* content = i->data;
3929 : char tmp[PATH_MAX];
3930 :
3931 : /* now renames the just written copy with the correct name */
3932 1012 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3933 1012 : if (rename(tmp, content->content) != 0) {
3934 : /* LCOV_EXCL_START */
3935 : log_fatal("Error renaming the content file '%s' to '%s' in rename(). %s.\n", tmp, content->content, strerror(errno));
3936 : exit(EXIT_FAILURE);
3937 : /* LCOV_EXCL_STOP */
3938 : }
3939 :
3940 1012 : i = i->next;
3941 : }
3942 151 : }
3943 :
3944 151 : void state_write(struct snapraid_state* state)
3945 : {
3946 : uint32_t crc;
3947 :
3948 : /* write all the content files */
3949 151 : state_write_content(state, &crc);
3950 :
3951 : /* verify the just written files */
3952 151 : state_verify_content(state, crc);
3953 :
3954 : /* rename the new files, over the old ones */
3955 151 : state_rename_content(state);
3956 :
3957 151 : state->need_write = 0; /* no write needed anymore */
3958 151 : state->checked_read = 0; /* what we wrote is not checked in read */
3959 151 : }
3960 :
3961 122 : void state_skip(struct snapraid_state* state)
3962 : {
3963 : tommy_node* i;
3964 :
3965 : /* for each disk */
3966 850 : for (i = state->disklist; i != 0; i = i->next) {
3967 : tommy_node* j;
3968 728 : struct snapraid_disk* disk = i->data;
3969 :
3970 728 : if (!disk->skip_access)
3971 727 : continue;
3972 :
3973 : /* for each file */
3974 1885 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
3975 1884 : struct snapraid_file* file = j->data;
3976 1884 : file_flag_set(file, FILE_IS_EXCLUDED);
3977 : }
3978 :
3979 : /* for each link */
3980 57 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
3981 56 : struct snapraid_link* slink = j->data;
3982 56 : link_flag_set(slink, FILE_IS_EXCLUDED);
3983 : }
3984 :
3985 : /* for each dir */
3986 1 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
3987 0 : struct snapraid_dir* dir = j->data;
3988 0 : dir_flag_set(dir, FILE_IS_EXCLUDED);
3989 : }
3990 : }
3991 122 : }
3992 :
3993 122 : void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing, int filter_error)
3994 : {
3995 : tommy_node* i;
3996 : unsigned l;
3997 :
3998 : /* if no filter, include all */
3999 122 : if (!filter_missing && !filter_error && tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk))
4000 235 : return;
4001 :
4002 9 : msg_progress("Filtering...\n");
4003 :
4004 17 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
4005 8 : struct snapraid_filter* filter = i->data;
4006 8 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_disk ? "//" : "");
4007 : }
4008 12 : for (i = tommy_list_head(filterlist_file); i != 0; i = i->next) {
4009 3 : struct snapraid_filter* filter = i->data;
4010 3 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_dir ? "/" : "");
4011 : }
4012 9 : if (filter_missing)
4013 2 : msg_verbose("\t<missing>\n");
4014 9 : if (filter_error)
4015 2 : msg_verbose("\t<error>\n");
4016 :
4017 : /* for each disk */
4018 63 : for (i = state->disklist; i != 0; i = i->next) {
4019 : tommy_node* j;
4020 54 : struct snapraid_disk* disk = i->data;
4021 :
4022 : /* if we filter for presence, we have to access the disk, so better to print something */
4023 54 : if (filter_missing)
4024 12 : msg_progress("Scanning disk %s...\n", disk->name);
4025 :
4026 : /* for each file */
4027 133944 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
4028 133890 : struct snapraid_file* file = j->data;
4029 :
4030 133890 : if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0
4031 80982 : || filter_path(filterlist_file, 0, disk->name, file->sub) != 0
4032 78138 : || filter_existence(filter_missing, disk->dir, file->sub) != 0
4033 38646 : || filter_correctness(filter_error, &state->infoarr, disk, file) != 0
4034 : ) {
4035 109458 : file_flag_set(file, FILE_IS_EXCLUDED);
4036 : }
4037 : }
4038 :
4039 : /* for each link */
4040 4414 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
4041 4360 : struct snapraid_link* slink = j->data;
4042 :
4043 4360 : if (filter_path(filterlist_disk, 0, disk->name, slink->sub) != 0
4044 2641 : || filter_path(filterlist_file, 0, disk->name, slink->sub) != 0
4045 2536 : || filter_existence(filter_missing, disk->dir, slink->sub) != 0
4046 : ) {
4047 3221 : link_flag_set(slink, FILE_IS_EXCLUDED);
4048 : }
4049 : }
4050 :
4051 : /* for each empty dir */
4052 82 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
4053 28 : struct snapraid_dir* dir = j->data;
4054 :
4055 28 : if (filter_emptydir(filterlist_disk, 0, disk->name, dir->sub) != 0
4056 17 : || filter_emptydir(filterlist_file, 0, disk->name, dir->sub) != 0
4057 17 : || filter_existence(filter_missing, disk->dir, dir->sub) != 0
4058 : ) {
4059 19 : dir_flag_set(dir, FILE_IS_EXCLUDED);
4060 : }
4061 : }
4062 : }
4063 :
4064 : /* if we are filtering by disk, exclude any parity not explicitely included */
4065 9 : if (!tommy_list_empty(filterlist_disk)) {
4066 : /* for each parity disk */
4067 20 : for (l = 0; l < state->level; ++l) {
4068 : /* check if the parity is excluded by name */
4069 16 : if (filter_path(filterlist_disk, 0, lev_config_name(l), 0) != 0) {
4070 : /* excluded the parity from further operation */
4071 11 : state->parity[l].is_excluded_by_filter = 1;
4072 : }
4073 : }
4074 : } else {
4075 : /* if we are filtering by file, exclude all parity */
4076 5 : if (filter_missing || !tommy_list_empty(filterlist_file)) {
4077 : /* for each parity disk */
4078 16 : for (l = 0; l < state->level; ++l) {
4079 13 : state->parity[l].is_excluded_by_filter = 1;
4080 : }
4081 : }
4082 : }
4083 : }
4084 :
4085 221 : int state_progress_begin(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, block_off_t countmax)
4086 : {
4087 : time_t now;
4088 :
4089 221 : if (state->opt.gui) {
4090 5 : log_tag("run:begin:%u:%u:%u\n", blockstart, blockmax, countmax);
4091 5 : log_flush();
4092 : }
4093 :
4094 221 : now = time(0);
4095 :
4096 221 : state->progress_whole_start = now;
4097 :
4098 221 : state->progress_tick = 0;
4099 221 : state->progress_ptr = 0;
4100 221 : state->progress_wasted = 0;
4101 :
4102 : /* stop if requested */
4103 221 : if (global_interrupt) {
4104 : /* LCOV_EXCL_START */
4105 : if (!state->opt.gui) {
4106 : msg_status("Not starting for interruption\n");
4107 : }
4108 : log_tag("sigint:0: SIGINT received\n");
4109 : log_flush();
4110 : return 0;
4111 : /* LCOV_EXCL_STOP */
4112 : }
4113 :
4114 221 : return 1;
4115 : }
4116 :
4117 221 : void state_progress_end(struct snapraid_state* state, block_off_t countpos, block_off_t countmax, data_off_t countsize)
4118 : {
4119 221 : if (state->opt.gui) {
4120 5 : log_tag("run:end\n");
4121 5 : log_flush();
4122 216 : } else if (countmax == 0) {
4123 19 : msg_status("Nothing to do\n");
4124 : } else {
4125 : time_t now;
4126 : time_t elapsed;
4127 :
4128 197 : unsigned countsize_MB = (countsize + MEGA - 1) / MEGA;
4129 :
4130 197 : now = time(0);
4131 :
4132 197 : elapsed = now - state->progress_whole_start - state->progress_wasted;
4133 :
4134 197 : msg_bar("%u%% completed, %u MB accessed", countpos * 100 / countmax, countsize_MB);
4135 :
4136 197 : msg_bar(" in %u:%02u", (unsigned)(elapsed / 3600), (unsigned)((elapsed % 3600) / 60));
4137 :
4138 : /* some extra spaces to ensure to overwrite the latest status line */
4139 197 : msg_bar(" ");
4140 :
4141 197 : msg_bar("\n");
4142 197 : msg_flush();
4143 : }
4144 221 : }
4145 :
4146 1 : void state_progress_stop(struct snapraid_state* state)
4147 : {
4148 : time_t now;
4149 :
4150 1 : now = time(0);
4151 :
4152 1 : if (!state->opt.gui) {
4153 1 : msg_bar("\n");
4154 1 : msg_flush();
4155 : }
4156 :
4157 1 : state->progress_interruption = now;
4158 1 : }
4159 :
4160 1 : void state_progress_restart(struct snapraid_state* state)
4161 : {
4162 : time_t now;
4163 :
4164 1 : now = time(0);
4165 :
4166 : /* reset the progress counter */
4167 1 : state->progress_tick = 0;
4168 1 : state->progress_ptr = 0;
4169 :
4170 1 : if (now >= state->progress_interruption) /* avoid degenerated cases when the clock is manually adjusted */
4171 1 : state->progress_wasted += now - state->progress_interruption;
4172 1 : }
4173 :
4174 : #define PROGRESS_CLEAR " "
4175 :
4176 : /**
4177 : * Set the latest tick data in the progress vector.
4178 : */
4179 478 : static void state_progress_latest(struct snapraid_state* state)
4180 : {
4181 : tommy_node* i;
4182 : unsigned l;
4183 :
4184 478 : state->progress_tick_misc[state->progress_ptr] = state->tick_misc;
4185 478 : state->progress_tick_sched[state->progress_ptr] = state->tick_sched;
4186 478 : state->progress_tick_raid[state->progress_ptr] = state->tick_raid;
4187 478 : state->progress_tick_hash[state->progress_ptr] = state->tick_hash;
4188 478 : state->progress_tick_io[state->progress_ptr] = state->tick_io;
4189 3338 : for (i = state->disklist; i != 0; i = i->next) {
4190 2860 : struct snapraid_disk* disk = i->data;
4191 2860 : disk->progress_tick[state->progress_ptr] = disk->tick;
4192 : }
4193 2711 : for (l = 0; l < state->level; ++l)
4194 2233 : state->parity[l].progress_tick[state->progress_ptr] = state->parity[l].tick;
4195 478 : }
4196 :
4197 : /**
4198 : * Number of columns
4199 : */
4200 : #define GRAPH_COLUMN 68
4201 :
4202 : /**
4203 : * Get the reference value, 0 if index is PROGRESS_MAX
4204 : */
4205 : #define ref(map, index) (index < PROGRESS_MAX ? map[index] : 0)
4206 :
4207 1 : static void state_progress_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest)
4208 : {
4209 : uint64_t v;
4210 : uint64_t tick_total;
4211 : unsigned bar;
4212 : tommy_node* i;
4213 : unsigned l;
4214 : size_t pad;
4215 :
4216 1 : tick_total = 0;
4217 :
4218 1 : tick_total += state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4219 1 : tick_total += state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4220 1 : tick_total += state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4221 1 : tick_total += state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4222 1 : tick_total += state->progress_tick_io[current] - ref(state->progress_tick_io, oldest);
4223 1 : if (!tick_total)
4224 1 : return;
4225 :
4226 1 : pad = 4;
4227 7 : for (i = state->disklist; i != 0; i = i->next) {
4228 6 : struct snapraid_disk* disk = i->data;
4229 6 : size_t len = strlen(disk->name);
4230 6 : if (pad < len)
4231 1 : pad = len;
4232 : }
4233 7 : for (l = 0; l < state->level; ++l) {
4234 6 : size_t len = strlen(lev_config_name(l));
4235 6 : if (pad < len)
4236 2 : pad = len;
4237 : }
4238 :
4239 : /* extra space */
4240 1 : pad += 1;
4241 :
4242 1 : if (pad + 30 < GRAPH_COLUMN)
4243 1 : bar = GRAPH_COLUMN - pad;
4244 : else
4245 0 : bar = 30;
4246 :
4247 1 : if (io) {
4248 0 : const char* legend = "cached blocks (instant, more is better)";
4249 :
4250 : /* refresh the cached blocks info */
4251 0 : io_refresh(io);
4252 :
4253 0 : printf("\n");
4254 :
4255 0 : for (i = state->disklist; i != 0; i = i->next) {
4256 0 : struct snapraid_disk* disk = i->data;
4257 0 : v = disk->cached;
4258 0 : printr(disk->name, pad);
4259 0 : printf("%4" PRIu64 " | ", v);
4260 0 : printc('o', v * bar / io->io_max);
4261 0 : printf("\n");
4262 : }
4263 :
4264 0 : for (l = 0; l < state->level; ++l) {
4265 0 : v = state->parity[l].cached;
4266 0 : printr(lev_config_name(l), pad);
4267 0 : printf("%4" PRIu64 " | ", v);
4268 0 : printc('o', v * bar / io->io_max);
4269 0 : printf("\n");
4270 : }
4271 :
4272 0 : printc(' ', pad);
4273 0 : printf(" |_");
4274 0 : printc('_', bar);
4275 0 : printf("\n");
4276 :
4277 0 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4278 0 : printf("%s", legend);
4279 0 : printf("\n");
4280 : }
4281 :
4282 1 : printf("\n");
4283 :
4284 7 : for (i = state->disklist; i != 0; i = i->next) {
4285 6 : struct snapraid_disk* disk = i->data;
4286 6 : v = disk->progress_tick[current] - ref(disk->progress_tick, oldest);
4287 6 : printr(disk->name, pad);
4288 6 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4289 6 : printc('*', v * bar / tick_total);
4290 6 : printf("\n");
4291 : }
4292 :
4293 7 : for (l = 0; l < state->level; ++l) {
4294 6 : v = state->parity[l].progress_tick[current] - ref(state->parity[l].progress_tick, oldest);
4295 6 : printr(lev_config_name(l), pad);
4296 6 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4297 6 : printc('*', v * bar / tick_total);
4298 6 : printf("\n");
4299 : }
4300 :
4301 1 : v = state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4302 1 : printr("raid", pad);
4303 1 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4304 1 : printc('*', v * bar / tick_total);
4305 1 : printf("\n");
4306 :
4307 1 : v = state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4308 1 : printr("hash", pad);
4309 1 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4310 1 : printc('*', v * bar / tick_total);
4311 1 : printf("\n");
4312 :
4313 1 : v = state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4314 1 : printr("sched", pad);
4315 1 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4316 1 : printc('*', v * bar / tick_total);
4317 1 : printf("\n");
4318 :
4319 1 : v = state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4320 1 : printr("misc", pad);
4321 1 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4322 1 : printc('*', v * bar / tick_total);
4323 1 : printf("\n");
4324 :
4325 1 : printc(' ', pad);
4326 1 : printf(" |_");
4327 1 : printc('_', bar);
4328 1 : printf("\n");
4329 :
4330 1 : if (oldest == PROGRESS_MAX) {
4331 1 : const char* legend = "wait time (total, less is better)";
4332 1 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4333 1 : printf("%s", legend);
4334 1 : printf("\n");
4335 : } else {
4336 0 : const char* legend_d = "wait time (last %d secs, less is better)";
4337 0 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend_d) / 2);
4338 0 : printf(legend_d, PROGRESS_MAX);
4339 0 : printf("\n");
4340 : }
4341 :
4342 1 : printf("\n");
4343 : }
4344 :
4345 1073492 : int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize)
4346 : {
4347 : time_t now;
4348 : int pred;
4349 :
4350 1073492 : now = time(0);
4351 :
4352 : /* previous position */
4353 1073492 : pred = state->progress_ptr + PROGRESS_MAX - 1;
4354 1073492 : if (pred >= PROGRESS_MAX)
4355 1073293 : pred -= PROGRESS_MAX;
4356 :
4357 : /* if the previous measure is different */
4358 1073492 : if (state->progress_tick == 0
4359 1073293 : || state->progress_time[pred] != now
4360 : ) {
4361 : time_t elapsed;
4362 382 : unsigned out_perc = 0;
4363 382 : unsigned out_speed = 0;
4364 382 : unsigned out_cpu = 0;
4365 382 : unsigned out_eta = 0;
4366 382 : int out_computed = 0;
4367 :
4368 : /* store the new measure */
4369 382 : state->progress_time[state->progress_ptr] = now;
4370 382 : state->progress_pos[state->progress_ptr] = countpos;
4371 382 : state->progress_size[state->progress_ptr] = countsize;
4372 382 : state_progress_latest(state);
4373 :
4374 382 : elapsed = now - state->progress_whole_start - state->progress_wasted;
4375 :
4376 : /* completion percentage */
4377 382 : if (countmax)
4378 382 : out_perc = countpos * 100 / countmax;
4379 :
4380 : /* if we have at least 5 measures */
4381 382 : if (state->progress_tick >= 5
4382 : /* or if we are running in test mode, with at least one measure */
4383 365 : || (state->opt.force_progress && state->progress_tick >= 1)
4384 : ) {
4385 : int oldest;
4386 : int past;
4387 : time_t delta_time;
4388 : block_off_t delta_pos;
4389 : data_off_t delta_size;
4390 : uint64_t tick_cpu;
4391 : uint64_t tick_total;
4392 : uint64_t delta_tick_cpu;
4393 : uint64_t delta_tick_total;
4394 : uint64_t oldest_tick_cpu;
4395 : uint64_t oldest_tick_total;
4396 :
4397 : /* number of past measures */
4398 183 : past = state->progress_tick;
4399 :
4400 : /* drop the oldest ones, to promptly */
4401 : /* skip the startup phase */
4402 183 : past -= past / 5;
4403 :
4404 : /* check how much we can go in the past */
4405 183 : if (past >= PROGRESS_MAX - 1) {
4406 : /* the vector is filled, so we are already in position */
4407 : /* to get the possible oldest one */
4408 0 : oldest = state->progress_ptr + 1;
4409 : } else {
4410 : /* go backward the number of positions selected */
4411 183 : oldest = state->progress_ptr + PROGRESS_MAX - past;
4412 : }
4413 183 : if (oldest >= PROGRESS_MAX)
4414 183 : oldest -= PROGRESS_MAX;
4415 :
4416 366 : tick_cpu = state->progress_tick_misc[state->progress_ptr]
4417 183 : + state->progress_tick_sched[state->progress_ptr]
4418 183 : + state->progress_tick_raid[state->progress_ptr]
4419 183 : + state->progress_tick_hash[state->progress_ptr];
4420 183 : tick_total = tick_cpu + state->progress_tick_io[state->progress_ptr];
4421 :
4422 366 : oldest_tick_cpu = state->progress_tick_misc[oldest]
4423 183 : + state->progress_tick_sched[oldest]
4424 183 : + state->progress_tick_raid[oldest]
4425 183 : + state->progress_tick_hash[oldest];
4426 183 : oldest_tick_total = oldest_tick_cpu + state->progress_tick_io[oldest];
4427 :
4428 183 : delta_time = now - state->progress_time[oldest];
4429 183 : delta_pos = countpos - state->progress_pos[oldest];
4430 183 : delta_size = countsize - state->progress_size[oldest];
4431 183 : delta_tick_cpu = tick_cpu - oldest_tick_cpu;
4432 183 : delta_tick_total = tick_total - oldest_tick_total;
4433 :
4434 : /* estimate the speed in MB/s */
4435 183 : if (delta_time != 0)
4436 183 : out_speed = (unsigned)(delta_size / MEGA / delta_time);
4437 :
4438 : /* estimate the cpu usage percentage */
4439 183 : if (delta_tick_total != 0)
4440 77 : out_cpu = (unsigned)(delta_tick_cpu * 100U / delta_tick_total);
4441 :
4442 : /* estimate the remaining time in minutes */
4443 183 : if (delta_pos != 0)
4444 183 : out_eta = (countmax - countpos) * delta_time / (60 * delta_pos);
4445 :
4446 183 : if (state->opt.force_stats) {
4447 0 : os_clear();
4448 0 : state_progress_graph(state, io, state->progress_ptr, oldest);
4449 : }
4450 :
4451 : /* we have the output value */
4452 183 : out_computed = 1;
4453 : }
4454 :
4455 382 : if (state->opt.gui) {
4456 7 : log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_speed, out_cpu, (uint64_t)elapsed);
4457 7 : log_flush();
4458 : } else {
4459 375 : msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA));
4460 375 : if (out_computed) {
4461 181 : msg_bar(", %u MB/s", out_speed);
4462 181 : msg_bar(", CPU %u%%", out_cpu);
4463 181 : msg_bar(", %u:%02u ETA", out_eta / 60, out_eta % 60);
4464 : }
4465 375 : msg_bar("%s\r", PROGRESS_CLEAR);
4466 375 : msg_flush();
4467 : }
4468 :
4469 : /* next position to fill */
4470 382 : ++state->progress_ptr;
4471 382 : if (state->progress_ptr >= PROGRESS_MAX)
4472 0 : state->progress_ptr -= PROGRESS_MAX;
4473 :
4474 : /* one more measure */
4475 382 : ++state->progress_tick;
4476 : }
4477 :
4478 : /* stop if requested */
4479 1073492 : if (global_interrupt) {
4480 : /* LCOV_EXCL_START */
4481 : if (!state->opt.gui) {
4482 : log_fatal("\n");
4483 : log_fatal("Stopping for interruption at block %u\n", blockpos);
4484 : }
4485 : log_tag("sigint:%u: SIGINT received\n", blockpos);
4486 : log_flush();
4487 : return 1;
4488 : /* LCOV_EXCL_STOP */
4489 : }
4490 :
4491 1073492 : return 0;
4492 : }
4493 :
4494 104 : void state_usage_waste(struct snapraid_state* state)
4495 : {
4496 104 : uint64_t now = tick();
4497 :
4498 104 : state->tick_last = now;
4499 104 : }
4500 :
4501 1063636 : void state_usage_misc(struct snapraid_state* state)
4502 : {
4503 1063636 : uint64_t now = tick();
4504 1063636 : uint64_t delta = now - state->tick_last;
4505 :
4506 : /* increment the time spent in computations */
4507 1063636 : state->tick_misc += delta;
4508 :
4509 1063636 : state->tick_last = now;
4510 1063636 : }
4511 :
4512 149919 : void state_usage_sched(struct snapraid_state* state)
4513 : {
4514 149919 : uint64_t now = tick();
4515 149919 : uint64_t delta = now - state->tick_last;
4516 :
4517 : /* increment the time spent in computations */
4518 149919 : state->tick_sched += delta;
4519 :
4520 149919 : state->tick_last = now;
4521 149919 : }
4522 :
4523 140474 : void state_usage_raid(struct snapraid_state* state)
4524 : {
4525 140474 : uint64_t now = tick();
4526 140474 : uint64_t delta = now - state->tick_last;
4527 :
4528 : /* increment the time spent in computations */
4529 140474 : state->tick_raid += delta;
4530 :
4531 140474 : state->tick_last = now;
4532 140474 : }
4533 :
4534 777321 : void state_usage_hash(struct snapraid_state* state)
4535 : {
4536 777321 : uint64_t now = tick();
4537 777321 : uint64_t delta = now - state->tick_last;
4538 :
4539 : /* increment the time spent in computations */
4540 777321 : state->tick_hash += delta;
4541 :
4542 777321 : state->tick_last = now;
4543 777321 : }
4544 :
4545 912855 : void state_usage_disk(struct snapraid_state* state, struct snapraid_handle* handle_map, unsigned* waiting_map, unsigned waiting_mac)
4546 : {
4547 912855 : uint64_t now = tick();
4548 912855 : uint64_t delta = now - state->tick_last;
4549 : unsigned i;
4550 :
4551 : /* increment the time spent in the data disks */
4552 955269 : for (i = 0; i < waiting_mac; ++i) {
4553 42414 : struct snapraid_disk* disk = handle_map[waiting_map[i]].disk;
4554 :
4555 42414 : if (!disk)
4556 0 : continue;
4557 :
4558 42414 : disk->tick += delta;
4559 : }
4560 912855 : state->tick_io += delta;
4561 :
4562 912855 : state->tick_last = now;
4563 912855 : }
4564 :
4565 774521 : void state_usage_parity(struct snapraid_state* state, unsigned* waiting_map, unsigned waiting_mac)
4566 : {
4567 774521 : uint64_t now = tick();
4568 774521 : uint64_t delta = now - state->tick_last;
4569 : unsigned i;
4570 :
4571 : /* increment the time spent in the parity disk */
4572 854191 : for (i = 0; i < waiting_mac; ++i)
4573 79670 : state->parity[waiting_map[i]].tick += delta;
4574 774521 : state->tick_io += delta;
4575 :
4576 774521 : state->tick_last = now;
4577 774521 : }
4578 :
4579 96 : void state_usage_print(struct snapraid_state* state)
4580 : {
4581 : /* set the latest data */
4582 96 : state_progress_latest(state);
4583 :
4584 96 : if (msg_level < MSG_PROGRESS)
4585 191 : return;
4586 :
4587 : /* print a graph for it */
4588 1 : state_progress_graph(state, 0, state->progress_ptr, PROGRESS_MAX);
4589 : }
4590 :
4591 499 : void state_fscheck(struct snapraid_state* state, const char* ope)
4592 : {
4593 : tommy_node* i;
4594 :
4595 : /* check the file-system on all disks */
4596 3487 : for (i = state->disklist; i != 0; i = i->next) {
4597 2988 : struct snapraid_disk* disk = i->data;
4598 :
4599 2988 : if (fs_check(disk) != 0) {
4600 : /* LCOV_EXCL_START */
4601 : log_fatal("Internal inconsistency in file-system for disk '%s' %s\n", disk->name, ope);
4602 : os_abort();
4603 : /* LCOV_EXCL_STOP */
4604 : }
4605 : }
4606 499 : }
4607 :
4608 1 : void generate_configuration(const char* path)
4609 : {
4610 : struct snapraid_state state;
4611 : struct snapraid_content* content;
4612 : char esc_buffer[ESC_MAX];
4613 : unsigned l, s;
4614 : tommy_node* j;
4615 :
4616 1 : state_init(&state);
4617 :
4618 : /* mark that we are without a configuration file */
4619 1 : state.no_conf = 1;
4620 :
4621 : /* create the dummy content entry */
4622 1 : content = content_alloc(path, -1);
4623 :
4624 : /* adds the content entry */
4625 1 : tommy_list_insert_tail(&state.contentlist, &content->node, content);
4626 :
4627 : /* read the content file */
4628 1 : state_read(&state);
4629 :
4630 : /* output a dummy configuration file */
4631 1 : printf("# Configuration file generated from %s\n", path);
4632 1 : printf("\n");
4633 1 : printf("# Use this blocksize\n");
4634 1 : printf("blocksize %u\n", state.block_size / KIBI);
4635 1 : printf("\n");
4636 1 : printf("# Use this hashsize\n");
4637 1 : printf("hashsize %u\n", BLOCK_HASH_SIZE);
4638 1 : printf("\n");
4639 7 : for (l = 0; l < state.level; ++l) {
4640 6 : printf("# Set the correct path for the %s files\n", lev_name(l));
4641 6 : printf("# You had %u of them:\n", state.parity[l].split_mac);
4642 30 : for (s = 0; s < state.parity[l].split_mac; ++s) {
4643 24 : printf("# %u:\n", s);
4644 24 : printf("# PATH:");
4645 24 : if (state.parity[l].split_map[s].path[0])
4646 24 : printf("%s", state.parity[l].split_map[s].path);
4647 : else
4648 0 : printf("?");
4649 24 : printf("\n");
4650 24 : printf("# SIZE:");
4651 24 : if (state.parity[l].split_map[s].size != PARITY_SIZE_INVALID)
4652 24 : printf("%" PRIu64, state.parity[l].split_map[s].size);
4653 : else
4654 0 : printf("?");
4655 24 : printf("\n");
4656 24 : printf("# UUID:");
4657 24 : if (state.parity[l].split_map[s].uuid[0])
4658 24 : printf("%s", state.parity[l].split_map[s].uuid);
4659 : else
4660 0 : printf("?");
4661 24 : printf("\n");
4662 24 : printf("#\n");
4663 : }
4664 6 : printf("%s ENTER_HERE_THE_PARITY_FILES_COMMA_SEPARATED\n", lev_config_name(l));
4665 6 : printf("\n");
4666 : }
4667 1 : printf("# Add any other content file\n");
4668 1 : printf("content %s\n", path);
4669 1 : printf("\n");
4670 7 : for (j = state.maplist; j; j = j->next) {
4671 6 : struct snapraid_map* map = j->data;
4672 : struct snapraid_disk* disk;
4673 6 : printf("# Set the correct dir for disk '%s'\n", map->name);
4674 6 : if (map->uuid[0])
4675 6 : printf("# Disk '%s' is the one with id '%s'\n", map->name, map->uuid);
4676 6 : disk = find_disk_by_name(&state, map->name);
4677 6 : if (disk && disk->filelist) {
4678 6 : struct snapraid_file* file = disk->filelist->data;
4679 6 : if (file) {
4680 6 : printf("# and containing: %s\n", fmt_poll(disk, file->sub, esc_buffer));
4681 : }
4682 : }
4683 6 : printf("data %s ENTER_HERE_THE_DIR\n", map->name);
4684 6 : printf("\n");
4685 : }
4686 :
4687 1 : state_done(&state);
4688 1 : }
4689 :
|