Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "elem.h"
21 : #include "import.h"
22 : #include "search.h"
23 : #include "state.h"
24 : #include "support.h"
25 : #include "parity.h"
26 : #include "stream.h"
27 : #include "handle.h"
28 : #include "io.h"
29 : #include "raid/raid.h"
30 : #include "raid/cpu.h"
31 :
32 : /**
33 : * Configure the multithread support.
34 : *
35 : * Multi thread for write could be either faster or slower, depending
36 : * on the specific conditions. With multithreads it's likely faster
37 : * writing to disk, but you'll need to access multiple times the same data,
38 : * being potentially slower.
39 : *
40 : * For upcoming SnapRAID version it's planned to add a mutex protection
41 : * at the file-system structure, slowing down multiple data access,
42 : * so we disable it.
43 : *
44 : * Multi thread for verify is instead always generally faster,
45 : * so we enable it if possible.
46 : */
47 : #if HAVE_PTHREAD
48 : /* #define HAVE_MT_WRITE 1 */
49 : #define HAVE_MT_VERIFY 1
50 : #endif
51 :
52 7 : const char* lev_name(unsigned l)
53 : {
54 7 : switch (l) {
55 2 : case 0 : return "Parity";
56 1 : case 1 : return "2-Parity";
57 1 : case 2 : return "3-Parity";
58 1 : case 3 : return "4-Parity";
59 1 : case 4 : return "5-Parity";
60 1 : case 5 : return "6-Parity";
61 : }
62 :
63 0 : return 0;
64 : }
65 :
66 186623 : const char* lev_config_name(unsigned l)
67 : {
68 186623 : switch (l) {
69 86887 : case 0 : return "parity";
70 60630 : case 1 : return "2-parity";
71 9816 : case 2 : return "3-parity";
72 9791 : case 3 : return "4-parity";
73 9766 : case 4 : return "5-parity";
74 9733 : case 5 : return "6-parity";
75 : }
76 :
77 0 : return 0;
78 : }
79 :
80 6345 : static int lev_config_scan(const char* s, unsigned* level, unsigned* mode)
81 : {
82 6345 : if (strcmp(s, "parity") == 0 || strcmp(s, "1-parity") == 0) {
83 449 : *level = 0;
84 449 : return 0;
85 : }
86 :
87 5896 : if (strcmp(s, "q-parity") == 0 || strcmp(s, "2-parity") == 0) {
88 225 : *level = 1;
89 225 : return 0;
90 : }
91 :
92 5671 : if (strcmp(s, "r-parity") == 0 || strcmp(s, "3-parity") == 0) {
93 200 : *level = 2;
94 200 : return 0;
95 : }
96 :
97 5471 : if (strcmp(s, "4-parity") == 0) {
98 197 : *level = 3;
99 197 : return 0;
100 : }
101 :
102 5274 : if (strcmp(s, "5-parity") == 0) {
103 194 : *level = 4;
104 194 : return 0;
105 : }
106 :
107 5080 : if (strcmp(s, "6-parity") == 0) {
108 190 : *level = 5;
109 190 : return 0;
110 : }
111 :
112 4890 : if (strcmp(s, "z-parity") == 0) {
113 0 : *level = 2;
114 0 : if (mode)
115 0 : *mode = RAID_MODE_VANDERMONDE;
116 0 : return 0;
117 : }
118 :
119 4890 : return -1;
120 : }
121 :
122 265 : const char* lev_raid_name(unsigned mode, unsigned n)
123 : {
124 265 : switch (n) {
125 41 : case 1 : return "par1";
126 25 : case 2 : return "par2";
127 3 : case 3 : if (mode == RAID_MODE_CAUCHY)
128 3 : return "par3";
129 : else
130 0 : return "parz";
131 3 : case 4 : return "par4";
132 4 : case 5 : return "par5";
133 189 : case 6 : return "par6";
134 : }
135 :
136 0 : return 0;
137 : }
138 :
139 267 : void state_init(struct snapraid_state* state)
140 : {
141 : unsigned l, s;
142 :
143 267 : memset(&state->opt, 0, sizeof(state->opt));
144 267 : state->filter_hidden = 0;
145 267 : state->autosave = 0;
146 267 : state->need_write = 0;
147 267 : state->checked_read = 0;
148 267 : state->block_size = 256 * KIBI; /* default 256 KiB */
149 267 : state->raid_mode = RAID_MODE_CAUCHY;
150 267 : state->file_mode = ADVISE_DEFAULT;
151 1869 : for (l = 0; l < LEV_MAX; ++l) {
152 1602 : state->parity[l].split_mac = 0;
153 14418 : for (s = 0; s < SPLIT_MAX; ++s) {
154 12816 : state->parity[l].split_map[s].path[0] = 0;
155 12816 : state->parity[l].split_map[s].uuid[0] = 0;
156 12816 : state->parity[l].split_map[s].size = PARITY_SIZE_INVALID;
157 12816 : state->parity[l].split_map[s].device = 0;
158 : }
159 1602 : state->parity[l].smartctl[0] = 0;
160 1602 : state->parity[l].total_blocks = 0;
161 1602 : state->parity[l].free_blocks = 0;
162 1602 : state->parity[l].skip_access = 0;
163 1602 : state->parity[l].tick = 0;
164 1602 : state->parity[l].cached = 0;
165 1602 : state->parity[l].is_excluded = 0;
166 : }
167 267 : state->tick_io = 0;
168 267 : state->tick_misc = 0;
169 267 : state->tick_sched = 0;
170 267 : state->tick_raid = 0;
171 267 : state->tick_hash = 0;
172 267 : state->tick_last = tick();
173 267 : state->share[0] = 0;
174 267 : state->pool[0] = 0;
175 267 : state->pool_device = 0;
176 267 : state->lockfile[0] = 0;
177 267 : state->level = 1; /* default is the lowest protection */
178 267 : state->clear_past_hash = 0;
179 267 : state->no_conf = 0;
180 :
181 267 : tommy_list_init(&state->disklist);
182 267 : tommy_list_init(&state->maplist);
183 267 : tommy_list_init(&state->contentlist);
184 267 : tommy_list_init(&state->filterlist);
185 267 : tommy_list_init(&state->importlist);
186 267 : tommy_hashdyn_init(&state->importset);
187 267 : tommy_hashdyn_init(&state->previmportset);
188 267 : tommy_hashdyn_init(&state->searchset);
189 267 : tommy_arrayblkof_init(&state->infoarr, sizeof(snapraid_info));
190 267 : }
191 :
192 244 : void state_done(struct snapraid_state* state)
193 : {
194 244 : tommy_list_foreach(&state->disklist, (tommy_foreach_func*)disk_free);
195 244 : tommy_list_foreach(&state->maplist, (tommy_foreach_func*)map_free);
196 244 : tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
197 244 : tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
198 244 : tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
199 244 : tommy_hashdyn_foreach(&state->searchset, (tommy_foreach_func*)search_file_free);
200 244 : tommy_hashdyn_done(&state->importset);
201 244 : tommy_hashdyn_done(&state->previmportset);
202 244 : tommy_hashdyn_done(&state->searchset);
203 244 : tommy_arrayblkof_done(&state->infoarr);
204 244 : }
205 :
206 : /**
207 : * Check the configuration.
208 : */
209 265 : static void state_config_check(struct snapraid_state* state, const char* path, tommy_list* filterlist_disk)
210 : {
211 : tommy_node* i;
212 : unsigned l, s;
213 :
214 : /* check for parity level */
215 265 : if (state->raid_mode == RAID_MODE_VANDERMONDE) {
216 0 : if (state->level > 3) {
217 : /* LCOV_EXCL_START */
218 : log_fatal("If you use the z-parity you cannot have more than 3 parities.\n");
219 : exit(EXIT_FAILURE);
220 : /* LCOV_EXCL_STIO */
221 : }
222 : }
223 :
224 : for (l = 0; l < state->level; ++l) {
225 : if (state->parity[l].split_mac == 0) {
226 : /* LCOV_EXCL_START */
227 : log_fatal("No '%s' specification in '%s'\n", lev_config_name(l), path);
228 : exit(EXIT_FAILURE);
229 : /* LCOV_EXCL_STOP */
230 : }
231 : }
232 :
233 265 : if (tommy_list_empty(&state->contentlist)) {
234 : /* LCOV_EXCL_START */
235 : log_fatal("No 'content' specification in '%s'\n", path);
236 : exit(EXIT_FAILURE);
237 : /* LCOV_EXCL_STOP */
238 : }
239 :
240 : /* check for equal paths */
241 1796 : for (i = state->contentlist; i != 0; i = i->next) {
242 1531 : struct snapraid_content* content = i->data;
243 :
244 9917 : for (l = 0; l < state->level; ++l) {
245 41930 : for (s = 0; s < state->parity[l].split_mac; ++s) {
246 33544 : if (pathcmp(state->parity[l].split_map[s].path, content->content) == 0) {
247 : /* LCOV_EXCL_START */
248 : log_fatal("Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
249 : exit(EXIT_FAILURE);
250 : /* LCOV_EXCL_STOP */
251 : }
252 : }
253 : }
254 : }
255 :
256 : /* check device of data disks */
257 265 : if (!state->opt.skip_device && !state->opt.skip_disk_access) {
258 0 : for (i = state->disklist; i != 0; i = i->next) {
259 : tommy_node* j;
260 0 : struct snapraid_disk* disk = i->data;
261 :
262 : /* skip data disks that are not accessible */
263 0 : if (disk->skip_access)
264 0 : continue;
265 :
266 : #ifdef _WIN32
267 : if (disk->device == 0) {
268 : /* LCOV_EXCL_START */
269 : log_fatal("Disk '%s' has a zero serial number.\n", disk->dir);
270 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
271 : log_fatal("it's better to change the serial number of the disk.\n");
272 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
273 : exit(EXIT_FAILURE);
274 : /* LCOV_EXCL_STOP */
275 : }
276 : #endif
277 :
278 0 : for (j = i->next; j != 0; j = j->next) {
279 0 : struct snapraid_disk* other = j->data;
280 0 : if (disk->device == other->device) {
281 0 : if (state->opt.force_device) {
282 : /* note tha we just ignore the issue */
283 : /* and we DON'T mark the disk to be skipped */
284 : /* because we want to use these disks */
285 0 : if (!state->opt.no_warnings)
286 0 : log_fatal("DANGER! Ignoring that disks '%s' and '%s' are on the same device\n", disk->name, other->name);
287 : } else {
288 : /* LCOV_EXCL_START */
289 : log_fatal("Disks '%s' and '%s' are on the same device.\n", disk->dir, other->dir);
290 : #ifdef _WIN32
291 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
292 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
293 : log_fatal("to change one of the disk serial.\n");
294 : #endif
295 : /* in "fix" we allow to continue anyway */
296 : if (strcmp(state->command, "fix") == 0) {
297 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
298 : }
299 : exit(EXIT_FAILURE);
300 : /* LCOV_EXCL_STOP */
301 : }
302 : }
303 : }
304 :
305 : /* skip data disks that are not accessible */
306 0 : if (disk->skip_access)
307 0 : continue;
308 :
309 0 : if (!state->opt.skip_parity_access) {
310 0 : for (l = 0; l < state->level; ++l) {
311 0 : for (s = 0; s < state->parity[l].split_mac; ++s) {
312 0 : if (disk->device == state->parity[l].split_map[s].device) {
313 : /* LCOV_EXCL_START */
314 : log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
315 : #ifdef _WIN32
316 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
317 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
318 : log_fatal("to change one of the disk serial.\n");
319 : #endif
320 : exit(EXIT_FAILURE);
321 : /* LCOV_EXCL_STOP */
322 : }
323 : }
324 : }
325 : }
326 :
327 0 : if (state->pool[0] != 0 && disk->device == state->pool_device) {
328 : /* LCOV_EXCL_START */
329 : log_fatal("Disk '%s' and pool '%s' are on the same device.\n", disk->dir, state->pool);
330 : #ifdef _WIN32
331 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
332 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
333 : log_fatal("to change one of the disk serial.\n");
334 : #endif
335 : exit(EXIT_FAILURE);
336 : /* LCOV_EXCL_STOP */
337 : }
338 : }
339 : }
340 :
341 : /* check device of parity disks */
342 265 : if (!state->opt.skip_device && !state->opt.skip_parity_access) {
343 0 : for (l = 0; l < state->level; ++l) {
344 0 : for (s = 0; s < state->parity[l].split_mac; ++s) {
345 : unsigned j,t;
346 :
347 : /* skip parity disks that are not accessible */
348 0 : if (state->parity[l].skip_access)
349 0 : continue;
350 :
351 : #ifdef _WIN32
352 : if (state->parity[l].split_map[s].device == 0) {
353 : /* LCOV_EXCL_START */
354 : log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].split_map[s].path);
355 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
356 : log_fatal("it's better to change the serial number of the disk.\n");
357 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
358 : exit(EXIT_FAILURE);
359 : /* LCOV_EXCL_STOP */
360 : }
361 : #endif
362 :
363 0 : for (j = l + 1; j < state->level; ++j) {
364 0 : for (t = 0; t < state->parity[j].split_mac; ++t) {
365 0 : if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
366 0 : if (state->opt.force_device) {
367 : /* note tha we just ignore the issue */
368 : /* and we DON'T mark the disk to be skipped */
369 : /* because we want to use these disks */
370 0 : if (!state->opt.no_warnings)
371 0 : log_fatal("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
372 : } else {
373 : /* LCOV_EXCL_START */
374 : log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].split_map[s].path, state->parity[j].split_map[t].path);
375 : #ifdef _WIN32
376 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].split_map[s].device);
377 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
378 : log_fatal("to change one of the disk serial.\n");
379 : #endif
380 : /* in "fix" we allow to continue anyway */
381 : if (strcmp(state->command, "fix") == 0) {
382 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
383 : }
384 : exit(EXIT_FAILURE);
385 : /* LCOV_EXCL_STOP */
386 : }
387 : }
388 : }
389 : }
390 :
391 0 : if (state->pool[0] != 0 && state->pool_device == state->parity[l].split_map[s].device) {
392 : /* LCOV_EXCL_START */
393 : log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].split_map[s].path);
394 : #ifdef _WIN32
395 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
396 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
397 : log_fatal("to change one of the disk serial.\n");
398 : #endif
399 : exit(EXIT_FAILURE);
400 : /* LCOV_EXCL_STOP */
401 : }
402 : }
403 : }
404 : }
405 :
406 : /* check device of pool disk */
407 : #ifdef _WIN32
408 : if (!state->opt.skip_device) {
409 : if (state->pool[0] != 0 && state->pool_device == 0) {
410 : /* LCOV_EXCL_START */
411 : log_fatal("Disk '%s' has a zero serial number.\n", state->pool);
412 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
413 : log_fatal("it's better to change the serial number of the disk.\n");
414 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
415 : exit(EXIT_FAILURE);
416 : /* LCOV_EXCL_STOP */
417 : }
418 : }
419 : #endif
420 :
421 : /* count the content files */
422 265 : if (!state->opt.skip_device && !state->opt.skip_content_access) {
423 : unsigned content_count;
424 :
425 0 : content_count = 0;
426 0 : for (i = state->contentlist; i != 0; i = i->next) {
427 : tommy_node* j;
428 0 : struct snapraid_content* content = i->data;
429 :
430 : /* check if there are others in the same disk */
431 0 : for (j = i->next; j != 0; j = j->next) {
432 0 : struct snapraid_content* other = j->data;
433 0 : if (content->device == other->device) {
434 0 : break;
435 : }
436 : }
437 0 : if (j != 0) {
438 : /* skip it */
439 0 : continue;
440 : }
441 :
442 0 : ++content_count;
443 : }
444 :
445 0 : if (content_count < state->level + 1) {
446 : /* LCOV_EXCL_START */
447 : log_fatal("You must have at least %d 'content' files in different disks.\n", state->level + 1);
448 : exit(EXIT_FAILURE);
449 : /* LCOV_EXCL_STOP */
450 : }
451 : }
452 :
453 : /* check for speed */
454 : #ifdef CONFIG_X86
455 265 : if (!raid_cpu_has_ssse3())
456 : #endif
457 0 : if (state->raid_mode == RAID_MODE_CAUCHY && !state->opt.no_warnings) {
458 0 : if (state->level == 3) {
459 0 : log_fatal("WARNING! Your CPU doesn't have a fast implementation for triple parity.\n");
460 0 : log_fatal("WARNING! It's recommended to switch to 'z-parity' instead than '3-parity'.\n");
461 0 : } else if (state->level > 3) {
462 0 : log_fatal("WARNING! Your CPU doesn't have a fast implementation beyond triple parity.\n");
463 0 : log_fatal("WARNING! It's recommended to reduce the parity levels to triple parity.\n");
464 : }
465 : }
466 :
467 : /* ensure that specified filter disks are valid ones */
468 275 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
469 : tommy_node* j;
470 10 : struct snapraid_filter* filter = i->data;
471 51 : for (j = state->disklist; j != 0; j = j->next) {
472 45 : struct snapraid_disk* disk = j->data;
473 45 : if (fnmatch(filter->pattern, disk->name, FNM_CASEINSENSITIVE_FOR_WIN) == 0)
474 4 : break;
475 : }
476 10 : if (j == 0) {
477 : /* check matching with parity disks */
478 21 : for (l = 0; l < state->level; ++l)
479 21 : if (fnmatch(filter->pattern, lev_config_name(l), FNM_CASEINSENSITIVE_FOR_WIN) == 0)
480 6 : break;
481 6 : if (l == state->level) {
482 : /* LCOV_EXCL_START */
483 : log_fatal("Option -d, --filter-disk %s doesn't match any data or parity disk.\n", filter->pattern);
484 : exit(EXIT_FAILURE);
485 : /* LCOV_EXCL_STOP */
486 : }
487 : }
488 : }
489 265 : }
490 :
491 : /**
492 : * Validate the smartctl command.
493 : *
494 : * It must contains only one %s string, and not other % chars.
495 : */
496 366 : static int validate_smartctl(const char* custom)
497 : {
498 366 : const char* s = custom;
499 366 : int arg = 0;
500 :
501 2562 : while (*s) {
502 1830 : if (s[0] == '%' && s[1] == 's') {
503 183 : if (arg) {
504 : /* LCOV_EXCL_START */
505 : return -1;
506 : /* LCOV_EXCL_STOP */
507 : }
508 183 : arg = 1;
509 1647 : } else if (s[0] == '%') {
510 : /* LCOV_EXCL_START */
511 : return -1;
512 : /* LCOV_EXCL_STOP */
513 : }
514 :
515 1830 : ++s;
516 : }
517 :
518 366 : return 0;
519 : }
520 :
521 266 : void state_config(struct snapraid_state* state, const char* path, const char* command, struct snapraid_option* opt, tommy_list* filterlist_disk)
522 : {
523 : STREAM* f;
524 : unsigned line;
525 : tommy_node* i;
526 : unsigned l, s;
527 :
528 : /* copy the options */
529 266 : state->opt = *opt;
530 :
531 : /* if unset, sort by physical order */
532 266 : if (!state->opt.force_order)
533 1 : state->opt.force_order = SORT_PHYSICAL;
534 :
535 : /* adjust file mode */
536 266 : if (state->opt.file_mode != ADVISE_DEFAULT) {
537 4 : state->file_mode = state->opt.file_mode;
538 : } else {
539 : /* default mode, if nothing is specified */
540 262 : state->file_mode = ADVISE_FLUSH;
541 : }
542 :
543 : /* store current command */
544 266 : state->command = command;
545 :
546 266 : log_tag("conf:file:%s\n", path);
547 :
548 266 : f = sopen_read(path);
549 266 : if (!f) {
550 : /* LCOV_EXCL_START */
551 : if (errno == ENOENT) {
552 : if (state->opt.auto_conf) {
553 : log_tag("conf:missing:\n");
554 : /* mark that we are without a configuration file */
555 : state->no_conf = 1;
556 : state->level = 0;
557 : return;
558 : }
559 :
560 : log_fatal("No configuration file found at '%s'\n", path);
561 : } else if (errno == EACCES) {
562 : log_fatal("You do not have rights to access the configuration file '%s'\n", path);
563 : } else {
564 : log_fatal("Error opening the configuration file '%s'. %s.\n", path, strerror(errno));
565 : }
566 : exit(EXIT_FAILURE);
567 : /* LCOV_EXCL_STOP */
568 : }
569 :
570 266 : line = 1;
571 : while (1) {
572 : char tag[PATH_MAX];
573 : char buffer[PATH_MAX];
574 : int ret;
575 : int c;
576 : unsigned level;
577 :
578 : /* skip initial spaces */
579 6245 : sgetspace(f);
580 :
581 : /* read the command */
582 6245 : ret = sgettok(f, tag, sizeof(tag));
583 6245 : if (ret < 0) {
584 : /* LCOV_EXCL_START */
585 : log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
586 : exit(EXIT_FAILURE);
587 : /* LCOV_EXCL_STOP */
588 : }
589 :
590 : /* skip spaces after the command */
591 6245 : sgetspace(f);
592 :
593 6245 : if (strcmp(tag, "blocksize") == 0
594 : /* block_size is the old format of the option */
595 5979 : || strcmp(tag, "block_size") == 0) {
596 :
597 266 : ret = sgetu32(f, &state->block_size);
598 266 : if (ret < 0) {
599 : /* LCOV_EXCL_START */
600 : log_fatal("Invalid 'blocksize' specification in '%s' at line %u\n", path, line);
601 : exit(EXIT_FAILURE);
602 : /* LCOV_EXCL_STOP */
603 : }
604 266 : if (state->block_size < 1) {
605 : /* LCOV_EXCL_START */
606 : log_fatal("Too small 'blocksize' specification in '%s' at line %u\n", path, line);
607 : exit(EXIT_FAILURE);
608 : /* LCOV_EXCL_STOP */
609 : }
610 266 : if (state->block_size > 16 * KIBI) {
611 : /* LCOV_EXCL_START */
612 : log_fatal("Too big 'blocksize' specification in '%s' at line %u\n", path, line);
613 : exit(EXIT_FAILURE);
614 : /* LCOV_EXCL_STOP */
615 : }
616 : /* check if it's a power of 2 */
617 266 : if ((state->block_size & (state->block_size - 1)) != 0) {
618 : /* LCOV_EXCL_START */
619 : log_fatal("Not power of 2 'blocksize' specification in '%s' at line %u\n", path, line);
620 : exit(EXIT_FAILURE);
621 : /* LCOV_EXCL_STOP */
622 : }
623 266 : state->block_size *= KIBI;
624 5979 : } else if (strcmp(tag, "hash_size") == 0) {
625 : uint32_t hash_size;
626 :
627 0 : ret = sgetu32(f, &hash_size);
628 0 : if (ret < 0) {
629 : /* LCOV_EXCL_START */
630 : log_fatal("Invalid 'hashsize' specification in '%s' at line %u\n", path, line);
631 : exit(EXIT_FAILURE);
632 : /* LCOV_EXCL_STOP */
633 : }
634 0 : if (hash_size < 2) {
635 : /* LCOV_EXCL_START */
636 : log_fatal("Too small 'hashsize' specification in '%s' at line %u\n", path, line);
637 : exit(EXIT_FAILURE);
638 : /* LCOV_EXCL_STOP */
639 : }
640 0 : if (hash_size > HASH_MAX) {
641 : /* LCOV_EXCL_START */
642 : log_fatal("Too big 'hashsize' specification in '%s' at line %u\n", path, line);
643 : exit(EXIT_FAILURE);
644 : /* LCOV_EXCL_STOP */
645 : }
646 : /* check if it's a power of 2 */
647 0 : if ((hash_size & (hash_size - 1)) != 0) {
648 : /* LCOV_EXCL_START */
649 : log_fatal("Not power of 2 'hashsize' specification in '%s' at line %u\n", path, line);
650 : exit(EXIT_FAILURE);
651 : /* LCOV_EXCL_STOP */
652 : }
653 :
654 0 : BLOCK_HASH_SIZE = hash_size;
655 5979 : } else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
656 : char device[PATH_MAX];
657 : char* split_map[SPLIT_MAX+1];
658 : unsigned split_mac;
659 : char* slash;
660 : uint64_t dev;
661 : int skip_access;
662 :
663 1272 : if (state->parity[level].split_mac != 0) {
664 : /* LCOV_EXCL_START */
665 : log_fatal("Multiple '%s' specification in '%s' at line %u\n", tag, path, line);
666 : exit(EXIT_FAILURE);
667 : /* LCOV_EXCL_STOP */
668 : }
669 :
670 1272 : ret = sgetlasttok(f, buffer, sizeof(buffer));
671 1272 : if (ret < 0) {
672 : /* LCOV_EXCL_START */
673 : log_fatal("Invalid '%s' specification in '%s' at line %u\n", tag, path, line);
674 : exit(EXIT_FAILURE);
675 : /* LCOV_EXCL_STOP */
676 : }
677 :
678 1272 : if (!*buffer) {
679 : /* LCOV_EXCL_START */
680 : log_fatal("Empty '%s' specification in '%s' at line %u\n", tag, path, line);
681 : exit(EXIT_FAILURE);
682 : /* LCOV_EXCL_STOP */
683 : }
684 :
685 1272 : split_mac = strsplit(split_map, SPLIT_MAX+1, buffer, ",");
686 :
687 1272 : if (split_mac > SPLIT_MAX) {
688 : /* LCOV_EXCL_START */
689 : log_fatal("Too many files in '%s' specification in '%s' at line %u\n", tag, path, line);
690 : exit(EXIT_FAILURE);
691 : /* LCOV_EXCL_STOP */
692 : }
693 :
694 1272 : skip_access = 0;
695 1272 : state->parity[level].split_mac = split_mac;
696 6356 : for (s = 0; s < split_mac; ++s) {
697 5085 : pathimport(state->parity[level].split_map[s].path, sizeof(state->parity[level].split_map[s].path), split_map[s]);
698 :
699 5085 : if (!state->opt.skip_parity_access) {
700 : struct stat st;
701 :
702 : /* get the device of the directory containing the parity file */
703 4349 : pathimport(device, sizeof(device), split_map[s]);
704 4349 : slash = strrchr(device, '/');
705 4349 : if (slash)
706 4349 : *slash = 0;
707 : else
708 0 : pathcpy(device, sizeof(device), ".");
709 :
710 4349 : if (stat(device, &st) == 0) {
711 4344 : dev = st.st_dev;
712 : } else {
713 : /* if the disk can be skipped */
714 5 : if (state->opt.force_device) {
715 : /* use a fake device, and mark the disk to be skipped */
716 4 : dev = 0;
717 4 : skip_access = 1;
718 4 : log_fatal("DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
719 : } else {
720 : /* LCOV_EXCL_START */
721 : log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
722 :
723 : /* in "fix" we allow to continue anyway */
724 : if (strcmp(state->command, "fix") == 0) {
725 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
726 : }
727 : exit(EXIT_FAILURE);
728 : /* LCOV_EXCL_STOP */
729 : }
730 : }
731 : } else {
732 : /* use a fake device */
733 736 : dev = 0;
734 : }
735 :
736 5084 : state->parity[level].split_map[s].device = dev;
737 : }
738 :
739 : /* store the global parity skip_access */
740 1271 : state->parity[level].skip_access = skip_access;
741 :
742 : /* adjust the level */
743 1271 : if (state->level < level + 1)
744 1005 : state->level = level + 1;
745 4707 : } else if (strcmp(tag, "share") == 0) {
746 41 : if (*state->share) {
747 : /* LCOV_EXCL_START */
748 : log_fatal("Multiple 'share' specification in '%s' at line %u\n", path, line);
749 : exit(EXIT_FAILURE);
750 : /* LCOV_EXCL_STOP */
751 : }
752 :
753 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
754 41 : if (ret < 0) {
755 : /* LCOV_EXCL_START */
756 : log_fatal("Invalid 'share' specification in '%s' at line %u\n", path, line);
757 : exit(EXIT_FAILURE);
758 : /* LCOV_EXCL_STOP */
759 : }
760 :
761 41 : if (!*buffer) {
762 : /* LCOV_EXCL_START */
763 : log_fatal("Empty 'share' specification in '%s' at line %u\n", path, line);
764 : exit(EXIT_FAILURE);
765 : /* LCOV_EXCL_STOP */
766 : }
767 :
768 41 : pathimport(state->share, sizeof(state->share), buffer);
769 4666 : } else if (strcmp(tag, "pool") == 0) {
770 : struct stat st;
771 :
772 41 : if (*state->pool) {
773 : /* LCOV_EXCL_START */
774 : log_fatal("Multiple 'pool' specification in '%s' at line %u\n", path, line);
775 : exit(EXIT_FAILURE);
776 : /* LCOV_EXCL_STOP */
777 : }
778 :
779 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
780 41 : if (ret < 0) {
781 : /* LCOV_EXCL_START */
782 : log_fatal("Invalid 'pool' specification in '%s' at line %u\n", path, line);
783 : exit(EXIT_FAILURE);
784 : /* LCOV_EXCL_STOP */
785 : }
786 :
787 41 : if (!*buffer) {
788 : /* LCOV_EXCL_START */
789 : log_fatal("Empty 'pool' specification in '%s' at line %u\n", path, line);
790 : exit(EXIT_FAILURE);
791 : /* LCOV_EXCL_STOP */
792 : }
793 :
794 41 : pathimport(state->pool, sizeof(state->pool), buffer);
795 :
796 : /* get the device of the directory containing the pool tree */
797 41 : if (stat(buffer, &st) != 0) {
798 : /* LCOV_EXCL_START */
799 : log_fatal("Error accessing 'pool' dir '%s' specification in '%s' at line %u\n", buffer, path, line);
800 : exit(EXIT_FAILURE);
801 : /* LCOV_EXCL_STOP */
802 : }
803 :
804 41 : state->pool_device = st.st_dev;
805 4625 : } else if (strcmp(tag, "content") == 0) {
806 : struct snapraid_content* content;
807 : char device[PATH_MAX];
808 : char* slash;
809 : struct stat st;
810 : uint64_t dev;
811 :
812 1531 : ret = sgetlasttok(f, buffer, sizeof(buffer));
813 1531 : if (ret < 0) {
814 : /* LCOV_EXCL_START */
815 : log_fatal("Invalid 'content' specification in '%s' at line %u\n", path, line);
816 : exit(EXIT_FAILURE);
817 : /* LCOV_EXCL_STOP */
818 : }
819 :
820 1531 : if (pathcmp(buffer, "/dev/null") == 0 || pathcmp(buffer, "NUL") == 0) {
821 : /* LCOV_EXCL_START */
822 : log_fatal("You cannot use the null device as 'content' specification in '%s' at line %u\n", path, line);
823 : exit(EXIT_FAILURE);
824 : /* LCOV_EXCL_STOP */
825 : }
826 :
827 1531 : if (!*buffer) {
828 : /* LCOV_EXCL_START */
829 : log_fatal("Empty 'content' specification in '%s' at line %u\n", path, line);
830 : exit(EXIT_FAILURE);
831 : /* LCOV_EXCL_STOP */
832 : }
833 :
834 : /* check if the content file is already specified */
835 5724 : for (i = state->contentlist; i != 0; i = i->next) {
836 4193 : content = i->data;
837 4193 : if (pathcmp(content->content, buffer) == 0)
838 0 : break;
839 : }
840 1531 : if (i) {
841 : /* LCOV_EXCL_START */
842 : log_fatal("Duplicate 'content' specification in '%s' at line %u\n", path, line);
843 : exit(EXIT_FAILURE);
844 : /* LCOV_EXCL_STOP */
845 : }
846 :
847 : /* get the device of the directory containing the content file */
848 1531 : pathimport(device, sizeof(device), buffer);
849 1531 : slash = strrchr(device, '/');
850 1531 : if (slash)
851 1531 : *slash = 0;
852 : else
853 0 : pathcpy(device, sizeof(device), ".");
854 1531 : if (stat(device, &st) == 0) {
855 1530 : dev = st.st_dev;
856 : } else {
857 1 : if (state->opt.skip_content_access) {
858 : /* use a fake device */
859 1 : dev = 0;
860 1 : log_fatal("WARNING! Skipping inaccessible content file '%s'...\n", buffer);
861 : } else {
862 : /* LCOV_EXCL_START */
863 : log_fatal("Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
864 : exit(EXIT_FAILURE);
865 : /* LCOV_EXCL_STOP */
866 : }
867 : }
868 :
869 : /* set the lock file at the first accessible content file */
870 1531 : if (state->lockfile[0] == 0 && dev != 0) {
871 265 : pathcpy(state->lockfile, sizeof(state->lockfile), buffer);
872 265 : pathcat(state->lockfile, sizeof(state->lockfile), ".lock");
873 : }
874 :
875 1531 : content = content_alloc(buffer, dev);
876 :
877 1531 : tommy_list_insert_tail(&state->contentlist, &content->node, content);
878 4679 : } else if (strcmp(tag, "data") == 0 || strcmp(tag, "disk") == 0) {
879 : /* "disk" is the deprecated name up to SnapRAID 9.x */
880 : char dir[PATH_MAX];
881 : char device[PATH_MAX];
882 : char uuid[UUID_MAX];
883 : struct snapraid_disk* disk;
884 : uint64_t dev;
885 : int skip_access;
886 :
887 1585 : ret = sgettok(f, buffer, sizeof(buffer));
888 1585 : if (ret < 0) {
889 : /* LCOV_EXCL_START */
890 : log_fatal("Invalid 'data' name specification in '%s' at line %u\n", path, line);
891 : exit(EXIT_FAILURE);
892 : /* LCOV_EXCL_STOP */
893 : }
894 :
895 1585 : if (!*buffer) {
896 : /* LCOV_EXCL_START */
897 : log_fatal("Empty 'data' name specification in '%s' at line %u\n", path, line);
898 : exit(EXIT_FAILURE);
899 : /* LCOV_EXCL_STOP */
900 : }
901 :
902 1585 : sgetspace(f);
903 :
904 1585 : ret = sgetlasttok(f, dir, sizeof(dir));
905 1585 : if (ret < 0) {
906 : /* LCOV_EXCL_START */
907 : log_fatal("Invalid 'data' dir specification in '%s' at line %u\n", path, line);
908 : exit(EXIT_FAILURE);
909 : /* LCOV_EXCL_STOP */
910 : }
911 :
912 1585 : if (!*dir) {
913 : /* LCOV_EXCL_START */
914 : log_fatal("Empty 'data' dir specification in '%s' at line %u\n", path, line);
915 : exit(EXIT_FAILURE);
916 : /* LCOV_EXCL_STOP */
917 : }
918 :
919 : /* get the device of the dir */
920 1585 : pathimport(device, sizeof(device), dir);
921 :
922 : /* check if the disk name already exists */
923 5535 : for (i = state->disklist; i != 0; i = i->next) {
924 3950 : disk = i->data;
925 3950 : if (strcmp(disk->name, buffer) == 0)
926 0 : break;
927 : }
928 1585 : if (i) {
929 : /* LCOV_EXCL_START */
930 : log_fatal("Duplicate 'data' name '%s' at line %u\n", buffer, line);
931 : exit(EXIT_FAILURE);
932 : /* LCOV_EXCL_STOP */
933 : }
934 :
935 : /* if the disk has to be present */
936 1585 : skip_access = 0;
937 1585 : if (!state->opt.skip_disk_access) {
938 : struct stat st;
939 :
940 1423 : if (stat(device, &st) == 0) {
941 1422 : dev = st.st_dev;
942 :
943 : /* read the uuid, if unsupported use an empty one */
944 1422 : if (devuuid(dev, uuid, sizeof(uuid)) != 0) {
945 0 : *uuid = 0;
946 : }
947 :
948 : /* fake a different UUID when testing */
949 1422 : if (state->opt.fake_uuid) {
950 2 : snprintf(uuid, sizeof(uuid), "fake-uuid-%d", state->opt.fake_uuid);
951 2 : --state->opt.fake_uuid;
952 : }
953 : } else {
954 : /* if the disk can be skipped */
955 1 : if (state->opt.force_device) {
956 : /* use a fake device, and mark the disk to be skipped */
957 1 : dev = 0;
958 1 : *uuid = 0;
959 1 : skip_access = 1;
960 1 : log_fatal("DANGER! Skipping inaccessible data disk '%s'...\n", buffer);
961 : } else {
962 : /* LCOV_EXCL_START */
963 : log_fatal("Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
964 :
965 : /* in "fix" we allow to continue anyway */
966 : if (strcmp(state->command, "fix") == 0) {
967 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
968 : }
969 : exit(EXIT_FAILURE);
970 : /* LCOV_EXCL_STOP */
971 : }
972 : }
973 : } else {
974 : /* use a fake device */
975 162 : dev = 0;
976 162 : *uuid = 0;
977 : }
978 :
979 1585 : disk = disk_alloc(buffer, dir, dev, uuid, skip_access);
980 :
981 1585 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
982 1509 : } else if (strcmp(tag, "smartctl") == 0) {
983 : char custom[PATH_MAX];
984 :
985 366 : ret = sgettok(f, buffer, sizeof(buffer));
986 366 : if (ret < 0) {
987 : /* LCOV_EXCL_START */
988 : log_fatal("Invalid 'smartctl' name specification in '%s' at line %u\n", path, line);
989 : exit(EXIT_FAILURE);
990 : /* LCOV_EXCL_STOP */
991 : }
992 :
993 366 : if (!*buffer) {
994 : /* LCOV_EXCL_START */
995 : log_fatal("Empty 'smartctl' name specification in '%s' at line %u\n", path, line);
996 : exit(EXIT_FAILURE);
997 : /* LCOV_EXCL_STOP */
998 : }
999 :
1000 366 : sgetspace(f);
1001 :
1002 366 : ret = sgetlasttok(f, custom, sizeof(custom));
1003 366 : if (ret < 0) {
1004 : /* LCOV_EXCL_START */
1005 : log_fatal("Invalid 'smartctl' option specification in '%s' at line %u\n", path, line);
1006 : exit(EXIT_FAILURE);
1007 : /* LCOV_EXCL_STOP */
1008 : }
1009 :
1010 366 : if (!*custom) {
1011 : /* LCOV_EXCL_START */
1012 : log_fatal("Empty 'smartctl' option specification in '%s' at line %u\n", path, line);
1013 : exit(EXIT_FAILURE);
1014 : /* LCOV_EXCL_STOP */
1015 : }
1016 :
1017 366 : if (validate_smartctl(custom) != 0) {
1018 : /* LCOV_EXCL_START */
1019 : log_fatal("Invalid 'smartctl' option specification in '%s' at line %u\n", path, line);
1020 : exit(EXIT_FAILURE);
1021 : /* LCOV_EXCL_STOP */
1022 : }
1023 :
1024 : /* search for parity */
1025 366 : if (lev_config_scan(buffer, &level, 0) == 0) {
1026 183 : if (state->parity[level].smartctl[0] != 0) {
1027 : /* LCOV_EXCL_START */
1028 : log_fatal("Duplicate parity smartctl '%s' at line %u\n", buffer, line);
1029 : exit(EXIT_FAILURE);
1030 : /* LCOV_EXCL_STOP */
1031 : }
1032 :
1033 183 : pathcpy(state->parity[level].smartctl, sizeof(state->parity[level].smartctl), custom);
1034 : } else {
1035 : struct snapraid_disk* disk;
1036 :
1037 : /* search the disk */
1038 183 : disk = 0;
1039 183 : for (i = state->disklist; i != 0; i = i->next) {
1040 183 : disk = i->data;
1041 183 : if (strcmp(disk->name, buffer) == 0)
1042 183 : break;
1043 : }
1044 183 : if (!disk) {
1045 : /* LCOV_EXCL_START */
1046 : log_fatal("Missing disk smartctl '%s' at line %u\n", buffer, line);
1047 : exit(EXIT_FAILURE);
1048 : /* LCOV_EXCL_STOP */
1049 : }
1050 183 : if (disk->smartctl[0] != 0) {
1051 : /* LCOV_EXCL_START */
1052 : log_fatal("Duplicate disk name '%s' at line %u\n", buffer, line);
1053 : exit(EXIT_FAILURE);
1054 : /* LCOV_EXCL_STOP */
1055 : }
1056 :
1057 183 : pathcpy(disk->smartctl, sizeof(disk->smartctl), custom);
1058 : }
1059 1143 : } else if (strcmp(tag, "nohidden") == 0) {
1060 0 : state->filter_hidden = 1;
1061 1143 : } else if (strcmp(tag, "exclude") == 0) {
1062 : struct snapraid_filter* filter;
1063 :
1064 265 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1065 265 : if (ret < 0) {
1066 : /* LCOV_EXCL_START */
1067 : log_fatal("Invalid 'exclude' specification in '%s' at line %u\n", path, line);
1068 : exit(EXIT_FAILURE);
1069 : /* LCOV_EXCL_STOP */
1070 : }
1071 :
1072 265 : if (!*buffer) {
1073 : /* LCOV_EXCL_START */
1074 : log_fatal("Empty 'exclude' specification in '%s' at line %u\n", path, line);
1075 : exit(EXIT_FAILURE);
1076 : /* LCOV_EXCL_STOP */
1077 : }
1078 :
1079 265 : filter = filter_alloc_file(-1, buffer);
1080 265 : if (!filter) {
1081 : /* LCOV_EXCL_START */
1082 : log_fatal("Invalid 'exclude' specification '%s' in '%s' at line %u\n", buffer, path, line);
1083 : log_fatal("Filters using relative paths are not supported. Ensure to add an initial slash\n");
1084 : exit(EXIT_FAILURE);
1085 : /* LCOV_EXCL_STOP */
1086 : }
1087 265 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1088 878 : } else if (strcmp(tag, "include") == 0) {
1089 : struct snapraid_filter* filter;
1090 :
1091 265 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1092 265 : if (ret < 0) {
1093 : /* LCOV_EXCL_START */
1094 : log_fatal("Invalid 'include' specification in '%s' at line %u\n", path, line);
1095 : exit(EXIT_FAILURE);
1096 : /* LCOV_EXCL_STOP */
1097 : }
1098 :
1099 265 : if (!*buffer) {
1100 : /* LCOV_EXCL_START */
1101 : log_fatal("Empty 'include' specification in '%s' at line %u\n", path, line);
1102 : exit(EXIT_FAILURE);
1103 : /* LCOV_EXCL_STOP */
1104 : }
1105 :
1106 265 : filter = filter_alloc_file(1, buffer);
1107 265 : if (!filter) {
1108 : /* LCOV_EXCL_START */
1109 : log_fatal("Invalid 'include' specification '%s' in '%s' at line %u\n", buffer, path, line);
1110 : log_fatal("Filters using relative paths are not supported. Ensure to add an initial slash\n");
1111 : exit(EXIT_FAILURE);
1112 : /* LCOV_EXCL_STOP */
1113 : }
1114 265 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1115 613 : } else if (strcmp(tag, "autosave") == 0) {
1116 : char* e;
1117 :
1118 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1119 41 : if (ret < 0) {
1120 : /* LCOV_EXCL_START */
1121 : log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1122 : exit(EXIT_FAILURE);
1123 : /* LCOV_EXCL_STOP */
1124 : }
1125 :
1126 41 : if (!*buffer) {
1127 : /* LCOV_EXCL_START */
1128 : log_fatal("Empty 'autosave' specification in '%s' at line %u\n", path, line);
1129 : exit(EXIT_FAILURE);
1130 : /* LCOV_EXCL_STOP */
1131 : }
1132 :
1133 41 : state->autosave = strtoul(buffer, &e, 0);
1134 :
1135 41 : if (!e || *e) {
1136 : /* LCOV_EXCL_START */
1137 : log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1138 : exit(EXIT_FAILURE);
1139 : /* LCOV_EXCL_STOP */
1140 : }
1141 :
1142 : /* convert to GB */
1143 41 : state->autosave *= GIGA;
1144 572 : } else if (tag[0] == 0) {
1145 : /* allow empty lines */
1146 41 : } else if (tag[0] == '#') {
1147 41 : ret = sgetline(f, buffer, sizeof(buffer));
1148 41 : if (ret < 0) {
1149 : /* LCOV_EXCL_START */
1150 : log_fatal("Invalid comment in '%s' at line %u\n", path, line);
1151 : exit(EXIT_FAILURE);
1152 : /* LCOV_EXCL_STOP */
1153 : }
1154 : } else {
1155 : /* LCOV_EXCL_START */
1156 : log_fatal("Invalid command '%s' in '%s' at line %u\n", tag, path, line);
1157 : exit(EXIT_FAILURE);
1158 : /* LCOV_EXCL_STOP */
1159 : }
1160 :
1161 : /* skip final spaces */
1162 6244 : sgetspace(f);
1163 :
1164 : /* next line */
1165 6244 : c = sgeteol(f);
1166 6244 : if (c == EOF) {
1167 265 : break;
1168 : }
1169 5979 : if (c != '\n') {
1170 : /* LCOV_EXCL_START */
1171 : log_fatal("Extra data in '%s' at line %u\n", path, line);
1172 : exit(EXIT_FAILURE);
1173 : /* LCOV_EXCL_STOP */
1174 : }
1175 5979 : ++line;
1176 5979 : }
1177 :
1178 265 : if (serror(f)) {
1179 : /* LCOV_EXCL_START */
1180 : log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
1181 : exit(EXIT_FAILURE);
1182 : /* LCOV_EXCL_STOP */
1183 : }
1184 :
1185 265 : sclose(f);
1186 :
1187 265 : state_config_check(state, path, filterlist_disk);
1188 :
1189 : /* select the default hash */
1190 265 : if (state->opt.force_murmur3) {
1191 1 : state->besthash = HASH_MURMUR3;
1192 264 : } else if (state->opt.force_spooky2) {
1193 1 : state->besthash = HASH_SPOOKY2;
1194 : } else {
1195 : #ifdef CONFIG_X86
1196 263 : if (sizeof(void*) == 4 && !raid_cpu_has_slowmult())
1197 263 : state->besthash = HASH_MURMUR3;
1198 : else
1199 0 : state->besthash = HASH_SPOOKY2;
1200 : #else
1201 : if (sizeof(void*) == 4)
1202 : state->besthash = HASH_MURMUR3;
1203 : else
1204 : state->besthash = HASH_SPOOKY2;
1205 : #endif
1206 : }
1207 :
1208 : /* by default use the best hash */
1209 265 : state->hash = state->besthash;
1210 :
1211 : /* by default use a random hash seed */
1212 265 : if (randomize(state->hashseed, HASH_MAX) != 0) {
1213 : /* LCOV_EXCL_START */
1214 : log_fatal("Failed to get random values.\n");
1215 : exit(EXIT_FAILURE);
1216 : /* LCOV_EXCL_STOP */
1217 : }
1218 :
1219 : /* no previous hash by default */
1220 265 : state->prevhash = HASH_UNDEFINED;
1221 :
1222 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
1223 :
1224 265 : log_tag("blocksize:%u\n", state->block_size);
1225 1850 : for (i = state->disklist; i != 0; i = i->next) {
1226 1585 : struct snapraid_disk* disk = i->data;
1227 1585 : log_tag("data:%s:%s\n", disk->name, disk->dir);
1228 : }
1229 :
1230 265 : log_tag("mode:%s\n", lev_raid_name(state->raid_mode, state->level));
1231 1531 : for (l = 0; l < state->level; ++l)
1232 6330 : for (s = 0; s < state->parity[l].split_mac; ++s)
1233 5064 : log_tag("%s:%u:%s\n", lev_config_name(l), s, state->parity[l].split_map[s].path);
1234 265 : if (state->pool[0] != 0)
1235 41 : log_tag("pool:%s\n", state->pool);
1236 265 : if (state->share[0] != 0)
1237 41 : log_tag("share:%s\n", state->share);
1238 265 : if (state->autosave != 0)
1239 41 : log_tag("autosave:%" PRIu64 "\n", state->autosave);
1240 795 : for (i = tommy_list_head(&state->filterlist); i != 0; i = i->next) {
1241 : char out[PATH_MAX];
1242 530 : struct snapraid_filter* filter = i->data;
1243 530 : log_tag("filter:%s\n", filter_type(filter, out, sizeof(out)));
1244 : }
1245 265 : if (state->filter_hidden)
1246 0 : log_tag("filter:nohidden:\n");
1247 265 : log_flush();
1248 : }
1249 :
1250 : /**
1251 : * Find a disk by name.
1252 : */
1253 7047 : static struct snapraid_disk* find_disk_by_name(struct snapraid_state* state, const char* name)
1254 : {
1255 : tommy_node* i;
1256 :
1257 24588 : for (i = state->disklist; i != 0; i = i->next) {
1258 24580 : struct snapraid_disk* disk = i->data;
1259 24580 : if (strcmp(disk->name, name) == 0)
1260 7039 : return disk;
1261 : }
1262 :
1263 8 : if (state->no_conf) {
1264 : /* without a configuration file, add disks automatically */
1265 : struct snapraid_disk* disk;
1266 :
1267 6 : disk = disk_alloc(name, "DUMMY/", -1, "", 0);
1268 :
1269 6 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
1270 :
1271 6 : return disk;
1272 : }
1273 :
1274 2 : return 0;
1275 : }
1276 :
1277 : /**
1278 : * Find a disk by UUID.
1279 : */
1280 2 : static struct snapraid_disk* find_disk_by_uuid(struct snapraid_state* state, const char* uuid)
1281 : {
1282 : tommy_node* i;
1283 2 : struct snapraid_disk* found = 0;
1284 :
1285 : /* special test case to find the first matching UUID */
1286 : /* when testing UUID are all equal or not supported */
1287 : /* and we should handle this case specifically */
1288 2 : if (state->opt.match_first_uuid)
1289 2 : return state->disklist->data;
1290 :
1291 : /* LCOV_EXCL_START */
1292 : /* never find an empty uuid */
1293 : if (!*uuid)
1294 : return 0;
1295 :
1296 : for (i = state->disklist; i != 0; i = i->next) {
1297 : struct snapraid_disk* disk = i->data;
1298 : if (strcmp(disk->uuid, uuid) == 0) {
1299 : /* never match duplicate UUID */
1300 : if (found)
1301 : return 0;
1302 : found = disk;
1303 : }
1304 : }
1305 :
1306 : return found;
1307 : /* LCOV_EXCL_STOP */
1308 : }
1309 :
1310 : /**
1311 : * Update the disk mapping if required.
1312 : */
1313 260 : static void state_map(struct snapraid_state* state)
1314 : {
1315 : unsigned hole;
1316 : tommy_node* i;
1317 : unsigned uuid_mismatch;
1318 : unsigned diskcount;
1319 : unsigned l, s;
1320 :
1321 : /* remove all the mapping without a disk */
1322 : /* this happens when a disk is removed from the configuration file */
1323 : /* From SnapRAID 4.0 mappings are automatically removed if a disk is not used */
1324 : /* when saving the content file, but we keep this code to import older content files. */
1325 1917 : for (i = state->maplist; i != 0; ) {
1326 1397 : struct snapraid_map* map = i->data;
1327 : struct snapraid_disk* disk;
1328 :
1329 1397 : disk = find_disk_by_name(state, map->name);
1330 :
1331 : /* go to the next mapping before removing */
1332 1397 : i = i->next;
1333 :
1334 1397 : if (disk == 0) {
1335 : /* disk not found, remove the mapping */
1336 0 : tommy_list_remove_existing(&state->maplist, &map->node);
1337 0 : map_free(map);
1338 : }
1339 : }
1340 :
1341 : /* maps each unmapped disk present in the configuration file in the first available hole */
1342 : /* this happens when you add disks for the first time in the configuration file */
1343 260 : hole = 0; /* first position to try */
1344 1815 : for (i = state->disklist; i != 0; i = i->next) {
1345 1555 : struct snapraid_disk* disk = i->data;
1346 : struct snapraid_map* map;
1347 : tommy_node* j;
1348 :
1349 : /* check if the disk is already mapped */
1350 5430 : for (j = state->maplist; j != 0; j = j->next) {
1351 5272 : map = j->data;
1352 5272 : if (strcmp(disk->name, map->name) == 0) {
1353 : /* mapping found */
1354 1397 : break;
1355 : }
1356 : }
1357 1555 : if (j != 0) {
1358 : /* mapping is present, then copy the free blocks into to disk */
1359 1397 : disk->total_blocks = map->total_blocks;
1360 1397 : disk->free_blocks = map->free_blocks;
1361 1397 : continue;
1362 : }
1363 :
1364 : /* mapping not found, search for an hole */
1365 : while (1) {
1366 1100 : for (j = state->maplist; j != 0; j = j->next) {
1367 942 : map = j->data;
1368 942 : if (map->position == hole) {
1369 : /* position already used */
1370 158 : break;
1371 : }
1372 : }
1373 316 : if (j == 0) {
1374 : /* hole found */
1375 158 : break;
1376 : }
1377 :
1378 : /* try with the next one */
1379 158 : ++hole;
1380 158 : }
1381 :
1382 : /* insert the new mapping */
1383 158 : map = map_alloc(disk->name, hole, 0, 0, "");
1384 :
1385 158 : tommy_list_insert_tail(&state->maplist, &map->node, map);
1386 : }
1387 :
1388 : /* without configuration don't check for number of data disks or uuid changes */
1389 260 : if (state->no_conf)
1390 261 : return;
1391 :
1392 : /* counter for the number of UUID mismatches */
1393 259 : uuid_mismatch = 0;
1394 :
1395 : /* check if mapping match the disk uuid */
1396 259 : if (!state->opt.skip_disk_access) {
1397 1626 : for (i = state->maplist; i != 0; i = i->next) {
1398 1393 : struct snapraid_map* map = i->data;
1399 : struct snapraid_disk* disk;
1400 :
1401 1393 : disk = find_disk_by_name(state, map->name);
1402 1393 : if (disk == 0) {
1403 : /* LCOV_EXCL_START */
1404 : log_fatal("Internal inconsistency for mapping '%s'\n", map->name);
1405 : os_abort();
1406 : /* LCOV_EXCL_STOP */
1407 : }
1408 :
1409 1393 : if (disk->has_unsupported_uuid) {
1410 : /* if uuid is not available, skip this one */
1411 1 : continue;
1412 : }
1413 :
1414 : /* if the uuid is changed */
1415 1392 : if (strcmp(disk->uuid, map->uuid) != 0) {
1416 : /* mark the disk as with an UUID change */
1417 113 : disk->has_different_uuid = 1;
1418 :
1419 : /* if the previous uuid is available */
1420 113 : if (map->uuid[0] != 0) {
1421 : /* count the number of uuid change */
1422 4 : ++uuid_mismatch;
1423 4 : log_fatal("UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, disk->uuid);
1424 : } else {
1425 : /* no message here, because having a disk without */
1426 : /* UUID is the normal state of an empty disk */
1427 109 : disk->had_empty_uuid = 1;
1428 : }
1429 :
1430 : /* update the uuid in the mapping, */
1431 113 : pathcpy(map->uuid, sizeof(map->uuid), disk->uuid);
1432 :
1433 : /* write the new state with the new uuid */
1434 113 : state->need_write = 1;
1435 : }
1436 : }
1437 : }
1438 :
1439 : /* check the parity uuid */
1440 259 : if (!state->opt.skip_parity_access) {
1441 1273 : for (l = 0; l < state->level; ++l) {
1442 5260 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1443 : char uuid[UUID_MAX];
1444 : int ret;
1445 :
1446 4208 : ret = devuuid(state->parity[l].split_map[s].device, uuid, sizeof(uuid));
1447 4208 : if (ret != 0) {
1448 : /* uuid not available, just ignore */
1449 4 : continue;
1450 : }
1451 :
1452 : /* if the uuid is changed */
1453 4204 : if (strcmp(uuid, state->parity[l].split_map[s].uuid) != 0) {
1454 : /* if the previous uuid is available */
1455 184 : if (state->parity[l].split_map[s].uuid[0] != 0) {
1456 : /* count the number of uuid change */
1457 0 : ++uuid_mismatch;
1458 0 : log_fatal("UUID change for parity '%s[%u]' from '%s' to '%s'\n", lev_config_name(l), s, state->parity[l].split_map[s].uuid, uuid);
1459 : }
1460 :
1461 : /* update the uuid */
1462 184 : pathcpy(state->parity[l].split_map[s].uuid, sizeof(state->parity[l].split_map[s].uuid), uuid);
1463 :
1464 : /* write the new state with the new uuid */
1465 184 : state->need_write = 1;
1466 : }
1467 : }
1468 : }
1469 : }
1470 :
1471 259 : if (!state->opt.force_uuid && uuid_mismatch > state->level) {
1472 : /* LCOV_EXCL_START */
1473 : log_fatal("Too many disks have UUID changed from the latest 'sync'.\n");
1474 : log_fatal("If this happens because you really replaced them,\n");
1475 : log_fatal("you can '%s' anyway, using 'snapraid --force-uuid %s'.\n", state->command, state->command);
1476 : log_fatal("Instead, it's possible that you messed up the disk mount points,\n");
1477 : log_fatal("and you have to restore the mount points at the state of the latest sync.\n");
1478 : exit(EXIT_FAILURE);
1479 : /* LCOV_EXCL_STOP */
1480 : }
1481 :
1482 : /* count the number of data disks, including holes left after removing some */
1483 259 : diskcount = 0;
1484 1808 : for (i = state->maplist; i != 0; i = i->next) {
1485 1549 : struct snapraid_map* map = i->data;
1486 :
1487 1549 : if (map->position + 1 > diskcount)
1488 1367 : diskcount = map->position + 1;
1489 : }
1490 :
1491 : /* ensure to don't go over the limit of the RAID engine */
1492 259 : if (diskcount > RAID_DATA_MAX) {
1493 : /* LCOV_EXCL_START */
1494 : log_fatal("Too many data disks. No more than %u.\n", RAID_DATA_MAX);
1495 : exit(EXIT_FAILURE);
1496 : /* LCOV_EXCL_STOP */
1497 : }
1498 :
1499 : /* now count the real number of data disks, excluding holes left after removing some */
1500 259 : diskcount = tommy_list_count(&state->maplist);
1501 :
1502 : /* recommend number of parities */
1503 259 : if (!state->opt.no_warnings) {
1504 1 : if (diskcount >= 36 && state->level < 6) {
1505 0 : log_fatal("WARNING! With %u disks it's recommended to use six parity levels.\n", diskcount);
1506 1 : } else if (diskcount >= 29 && state->level < 5) {
1507 0 : log_fatal("WARNING! With %u disks it's recommended to use five parity levels.\n", diskcount);
1508 1 : } else if (diskcount >= 22 && state->level < 4) {
1509 0 : log_fatal("WARNING! With %u disks it's recommended to use four parity levels.\n", diskcount);
1510 1 : } else if (diskcount >= 15 && state->level < 3) {
1511 0 : log_fatal("WARNING! With %u disks it's recommended to use three parity levels.\n", diskcount);
1512 1 : } else if (diskcount >= 5 && state->level < 2) {
1513 0 : log_fatal("WARNING! With %u disks it's recommended to use two parity levels.\n", diskcount);
1514 : }
1515 : }
1516 : }
1517 :
1518 173 : void state_refresh(struct snapraid_state* state)
1519 : {
1520 : tommy_node* i;
1521 : unsigned l, s;
1522 :
1523 : /* for all disks */
1524 1209 : for (i = state->maplist; i != 0; i = i->next) {
1525 1036 : struct snapraid_map* map = i->data;
1526 : struct snapraid_disk* disk;
1527 : uint64_t total_space;
1528 : uint64_t free_space;
1529 : int ret;
1530 :
1531 1036 : disk = find_disk_by_name(state, map->name);
1532 1036 : if (disk == 0) {
1533 : /* LCOV_EXCL_START */
1534 : log_fatal("Internal inconsistency for mapping '%s'\n", map->name);
1535 : os_abort();
1536 : /* LCOV_EXCL_STOP */
1537 : }
1538 :
1539 1036 : ret = fsinfo(disk->dir, 0, &total_space, &free_space);
1540 1036 : if (ret != 0) {
1541 : /* LCOV_EXCL_START */
1542 : log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
1543 : exit(EXIT_FAILURE);
1544 : /* LCOV_EXCL_STOP */
1545 : }
1546 :
1547 : /* set the new free blocks */
1548 1036 : map->total_blocks = total_space / state->block_size;
1549 1036 : map->free_blocks = free_space / state->block_size;
1550 :
1551 : /* also update the disk info */
1552 1036 : disk->total_blocks = map->total_blocks;
1553 1036 : disk->free_blocks = map->free_blocks;
1554 : }
1555 :
1556 : /* for all parities */
1557 1141 : for (l = 0; l < state->level; ++l) {
1558 : /* set the new free blocks */
1559 968 : state->parity[l].total_blocks = 0;
1560 968 : state->parity[l].free_blocks = 0;
1561 :
1562 4840 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1563 : uint64_t total_space;
1564 : uint64_t free_space;
1565 : int ret;
1566 :
1567 3872 : ret = fsinfo(state->parity[l].split_map[s].path, 0, &total_space, &free_space);
1568 3872 : if (ret != 0) {
1569 : /* LCOV_EXCL_START */
1570 : log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].split_map[s].path, strerror(errno));
1571 : exit(EXIT_FAILURE);
1572 : /* LCOV_EXCL_STOP */
1573 : }
1574 :
1575 : /* add the new free blocks */
1576 3872 : state->parity[l].total_blocks += total_space / state->block_size;
1577 3872 : state->parity[l].free_blocks += free_space / state->block_size;
1578 : }
1579 : }
1580 :
1581 : /* note what we don't set need_write = 1, because we don't want */
1582 : /* to update the content file only for the free space info. */
1583 173 : }
1584 :
1585 : /**
1586 : * Check the content.
1587 : */
1588 253 : static void state_content_check(struct snapraid_state* state, const char* path)
1589 : {
1590 : tommy_node* i;
1591 :
1592 : /* check that any map has different name and position */
1593 1766 : for (i = state->maplist; i != 0; i = i->next) {
1594 1513 : struct snapraid_map* map = i->data;
1595 : tommy_node* j;
1596 5283 : for (j = i->next; j != 0; j = j->next) {
1597 3770 : struct snapraid_map* other = j->data;
1598 3770 : if (strcmp(map->name, other->name) == 0) {
1599 : /* LCOV_EXCL_START */
1600 : log_fatal("Colliding 'map' disk specification in '%s'\n", path);
1601 : exit(EXIT_FAILURE);
1602 : /* LCOV_EXCL_STOP */
1603 : }
1604 3770 : if (map->position == other->position) {
1605 : /* LCOV_EXCL_START */
1606 : log_fatal("Colliding 'map' index specification in '%s'\n", path);
1607 : exit(EXIT_FAILURE);
1608 : /* LCOV_EXCL_STOP */
1609 : }
1610 : }
1611 : }
1612 253 : }
1613 :
1614 : /**
1615 : * Check if the position is REQUIRED, or we can completely clear it from the state.
1616 : *
1617 : * Note that position with only DELETED blocks are discarged.
1618 : */
1619 756099 : static int fs_position_is_required(struct snapraid_state* state, block_off_t pos)
1620 : {
1621 : tommy_node* i;
1622 :
1623 : /* check for each disk */
1624 802725 : for (i = state->disklist; i != 0; i = i->next) {
1625 802725 : struct snapraid_disk* disk = i->data;
1626 802725 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1627 :
1628 : /* if we have at least one file, the position is needed */
1629 802725 : if (block_has_file(block))
1630 756099 : return 1;
1631 : }
1632 :
1633 0 : return 0;
1634 : }
1635 :
1636 : /**
1637 : * Check if the info block is REQUIREQ.
1638 : *
1639 : * This is used to ensure that we keep the last check used for scrubbing.
1640 : * and that we add it when importing old context files.
1641 : *
1642 : * Note that you can have position without info blocks, for example
1643 : * if all the blocks are not synced.
1644 : *
1645 : * Note also that not requiring an info block, doesn't mean that if present it
1646 : * can be discarded.
1647 : */
1648 1547923 : static int fs_info_is_required(struct snapraid_state* state, block_off_t pos)
1649 : {
1650 : tommy_node* i;
1651 :
1652 : /* check for each disk */
1653 2048488 : for (i = state->disklist; i != 0; i = i->next) {
1654 1989723 : struct snapraid_disk* disk = i->data;
1655 1989723 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1656 :
1657 : /* if we have at least one synced file, the info is required */
1658 1989723 : if (block_state_get(block) == BLOCK_STATE_BLK)
1659 1489158 : return 1;
1660 : }
1661 :
1662 58765 : return 0;
1663 : }
1664 :
1665 0 : static void fs_position_clear_deleted(struct snapraid_state* state, block_off_t pos)
1666 : {
1667 : tommy_node* i;
1668 :
1669 : /* check for each disk if block is really used */
1670 0 : for (i = state->disklist; i != 0; i = i->next) {
1671 0 : struct snapraid_disk* disk = i->data;
1672 0 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1673 :
1674 : /* if the block is deleted */
1675 0 : if (block_state_get(block) == BLOCK_STATE_DELETED) {
1676 : /* set it to empty */
1677 0 : fs_deallocate(disk, pos);
1678 : }
1679 : }
1680 0 : }
1681 :
1682 : /**
1683 : * Check if a block position in a disk is deleted.
1684 : */
1685 4529621 : static int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
1686 : {
1687 4529621 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1688 :
1689 4529621 : return block_state_get(block) == BLOCK_STATE_DELETED;
1690 : }
1691 :
1692 : /**
1693 : * Flush the file checking the final CRC.
1694 : * We exploit the fact that the CRC is always stored in the last 4 bytes.
1695 : */
1696 1 : static void decoding_error(const char* path, STREAM* f)
1697 : {
1698 : unsigned char buf[4];
1699 : uint32_t crc_stored;
1700 : uint32_t crc_computed;
1701 :
1702 1 : if (seof(f)) {
1703 : /* LCOV_EXCL_START */
1704 : log_fatal("Unexpected end of content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1705 : log_fatal("This content file is truncated. Use an alternate copy.\n");
1706 : exit(EXIT_FAILURE);
1707 : /* LCOV_EXCL_STOP */
1708 : }
1709 :
1710 1 : if (serror(f)) {
1711 : /* LCOV_EXCL_START */
1712 : log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1713 : exit(EXIT_FAILURE);
1714 : /* LCOV_EXCL_STOP */
1715 : }
1716 :
1717 1 : log_fatal("Decoding error in '%s' at offset %" PRIi64 "\n", path, stell(f));
1718 :
1719 1 : if (sdeplete(f, buf) != 0) {
1720 : /* LCOV_EXCL_START */
1721 : log_fatal("Error flushing the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1722 : exit(EXIT_FAILURE);
1723 : /* LCOV_EXCL_STOP */
1724 : }
1725 :
1726 : /* get the stored crc from the last four bytes */
1727 1 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
1728 :
1729 : /* get the computed crc */
1730 1 : crc_computed = scrc(f);
1731 :
1732 : /* adjust the stored crc to include itself */
1733 1 : crc_stored = crc32c(crc_stored, buf, 4);
1734 :
1735 1 : if (crc_computed != crc_stored) {
1736 1 : log_fatal("Mismatching CRC in '%s'\n", path);
1737 1 : log_fatal("This content file is damaged! Use an alternate copy.\n");
1738 1 : exit(EXIT_FAILURE);
1739 : } else {
1740 0 : log_fatal("The file CRC is correct!\n");
1741 : }
1742 0 : }
1743 :
1744 254 : static void state_read_content(struct snapraid_state* state, const char* path, STREAM* f)
1745 : {
1746 : block_off_t blockmax;
1747 : unsigned count_file;
1748 : unsigned count_hardlink;
1749 : unsigned count_symlink;
1750 : unsigned count_dir;
1751 : int crc_checked;
1752 : char buffer[PATH_MAX];
1753 : int ret;
1754 : tommy_array disk_mapping;
1755 : uint32_t mapping_max;
1756 :
1757 254 : blockmax = 0;
1758 254 : count_file = 0;
1759 254 : count_hardlink = 0;
1760 254 : count_symlink = 0;
1761 254 : count_dir = 0;
1762 254 : crc_checked = 0;
1763 254 : mapping_max = 0;
1764 254 : tommy_array_init(&disk_mapping);
1765 :
1766 254 : ret = sread(f, buffer, 12);
1767 254 : if (ret < 0) {
1768 : /* LCOV_EXCL_START */
1769 : decoding_error(path, f);
1770 : log_fatal("Invalid header!\n");
1771 : os_abort();
1772 : /* LCOV_EXCL_STOP */
1773 : }
1774 :
1775 : /*
1776 : * File format versions:
1777 : * - SNAPCNT1/SnapRAID 4.0 First version.
1778 : * - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support.
1779 : * The previous 'm' entry is now deprecated, but supported for importing.
1780 : * Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
1781 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
1782 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
1783 : * The previous 'P' entry is now deprecated, but supported for importing.
1784 : */
1785 254 : if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
1786 254 : && memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
1787 254 : && memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
1788 : ) {
1789 : /* LCOV_EXCL_START */
1790 : if (memcmp(buffer, "SNAPCNT", 7) != 0) {
1791 : decoding_error(path, f);
1792 : log_fatal("Invalid header!\n");
1793 : os_abort();
1794 : } else {
1795 : log_fatal("The content file '%s' was generated with a newer version of SnapRAID!\n", path);
1796 : exit(EXIT_FAILURE);
1797 : }
1798 : /* LCOV_EXCL_STOP */
1799 : }
1800 :
1801 : while (1) {
1802 : int c;
1803 :
1804 : /* read the command */
1805 3735612 : c = sgetc(f);
1806 3735612 : if (c == EOF) {
1807 253 : break;
1808 : }
1809 :
1810 3735359 : if (c == 'f') {
1811 : /* file */
1812 : char sub[PATH_MAX];
1813 : uint64_t v_size;
1814 : uint64_t v_mtime_sec;
1815 : uint32_t v_mtime_nsec;
1816 : uint64_t v_inode;
1817 : uint32_t v_idx;
1818 : struct snapraid_file* file;
1819 : struct snapraid_disk* disk;
1820 : uint32_t mapping;
1821 :
1822 3611762 : ret = sgetb32(f, &mapping);
1823 3611762 : if (ret < 0 || mapping >= mapping_max) {
1824 : /* LCOV_EXCL_START */
1825 : decoding_error(path, f);
1826 : log_fatal("Internal inconsistency in mapping index!\n");
1827 : os_abort();
1828 : /* LCOV_EXCL_STOP */
1829 : }
1830 3611762 : disk = tommy_array_get(&disk_mapping, mapping);
1831 :
1832 3611762 : ret = sgetb64(f, &v_size);
1833 3611762 : if (ret < 0) {
1834 : /* LCOV_EXCL_START */
1835 : decoding_error(path, f);
1836 : os_abort();
1837 : /* LCOV_EXCL_STOP */
1838 : }
1839 :
1840 3611762 : if (state->block_size == 0) {
1841 : /* LCOV_EXCL_START */
1842 : decoding_error(path, f);
1843 : log_fatal("Internal incosistency due zero blocksize!\n");
1844 : exit(EXIT_FAILURE);
1845 : /* LCOV_EXCL_STOP */
1846 : }
1847 :
1848 : /* check for impossible file size to avoid to crash for a too big allocation */
1849 3611762 : if (v_size / state->block_size > blockmax) {
1850 : /* LCOV_EXCL_START */
1851 : decoding_error(path, f);
1852 : log_fatal("Internal inconsistency in file size too big!\n");
1853 : os_abort();
1854 : /* LCOV_EXCL_STOP */
1855 : }
1856 :
1857 3611762 : ret = sgetb64(f, &v_mtime_sec);
1858 3611762 : if (ret < 0) {
1859 : /* LCOV_EXCL_START */
1860 : decoding_error(path, f);
1861 : os_abort();
1862 : /* LCOV_EXCL_STOP */
1863 : }
1864 :
1865 3611762 : ret = sgetb32(f, &v_mtime_nsec);
1866 3611762 : if (ret < 0) {
1867 : /* LCOV_EXCL_START */
1868 : decoding_error(path, f);
1869 : os_abort();
1870 : /* LCOV_EXCL_STOP */
1871 : }
1872 :
1873 : /* STAT_NSEC_INVALID is encoded as 0 */
1874 3611762 : if (v_mtime_nsec == 0)
1875 0 : v_mtime_nsec = STAT_NSEC_INVALID;
1876 : else
1877 3611762 : --v_mtime_nsec;
1878 :
1879 3611762 : ret = sgetb64(f, &v_inode);
1880 3611762 : if (ret < 0) {
1881 : /* LCOV_EXCL_START */
1882 : decoding_error(path, f);
1883 : os_abort();
1884 : /* LCOV_EXCL_STOP */
1885 : }
1886 :
1887 3611762 : ret = sgetbs(f, sub, sizeof(sub));
1888 3611762 : if (ret < 0) {
1889 : /* LCOV_EXCL_START */
1890 : decoding_error(path, f);
1891 : os_abort();
1892 : /* LCOV_EXCL_STOP */
1893 : }
1894 3611762 : if (!*sub) {
1895 : /* LCOV_EXCL_START */
1896 : decoding_error(path, f);
1897 : log_fatal("Internal inconsistency for null file!\n");
1898 : os_abort();
1899 : /* LCOV_EXCL_STOP */
1900 : }
1901 :
1902 : /* allocate the file */
1903 3611762 : file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0);
1904 :
1905 : /* insert the file in the file containers */
1906 3611762 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
1907 3611762 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
1908 3611762 : tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
1909 3611762 : tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
1910 :
1911 : /* read all the blocks */
1912 3611762 : v_idx = 0;
1913 10855962 : while (v_idx < file->blockmax) {
1914 : block_off_t v_pos;
1915 : uint32_t v_count;
1916 :
1917 : /* get the "subcommand */
1918 3632438 : c = sgetc(f);
1919 :
1920 3632438 : ret = sgetb32(f, &v_pos);
1921 3632438 : if (ret < 0) {
1922 : /* LCOV_EXCL_START */
1923 : decoding_error(path, f);
1924 : os_abort();
1925 : /* LCOV_EXCL_STOP */
1926 : }
1927 :
1928 3632438 : ret = sgetb32(f, &v_count);
1929 3632438 : if (ret < 0) {
1930 : /* LCOV_EXCL_START */
1931 : decoding_error(path, f);
1932 : os_abort();
1933 : /* LCOV_EXCL_STOP */
1934 : }
1935 :
1936 3632438 : if (v_idx + v_count > file->blockmax) {
1937 : /* LCOV_EXCL_START */
1938 : decoding_error(path, f);
1939 : log_fatal("Internal inconsistency in block number!\n");
1940 : os_abort();
1941 : /* LCOV_EXCL_STOP */
1942 : }
1943 :
1944 3632438 : if (v_pos + v_count > blockmax) {
1945 : /* LCOV_EXCL_START */
1946 : decoding_error(path, f);
1947 : log_fatal("Internal inconsistency in block size %u/%u!\n", blockmax, v_pos + v_count);
1948 : os_abort();
1949 : /* LCOV_EXCL_START */
1950 : }
1951 :
1952 : /* fill the blocks in the run */
1953 : while (v_count) {
1954 : struct snapraid_block* block = fs_file2block_get(file, v_idx);
1955 :
1956 : switch (c) {
1957 : case 'b' :
1958 : block_state_set(block, BLOCK_STATE_BLK);
1959 : break;
1960 : case 'n' :
1961 : /* deprecated NEW blocks are converted to CHG ones */
1962 : block_state_set(block, BLOCK_STATE_CHG);
1963 : break;
1964 : case 'g' :
1965 : block_state_set(block, BLOCK_STATE_CHG);
1966 : break;
1967 : case 'p' :
1968 : block_state_set(block, BLOCK_STATE_REP);
1969 : break;
1970 : default :
1971 : /* LCOV_EXCL_START */
1972 : decoding_error(path, f);
1973 : log_fatal("Invalid block type!\n");
1974 : os_abort();
1975 : /* LCOV_EXCL_STOP */
1976 : }
1977 :
1978 : /* read the hash only for 'blk/chg/rep', and not for 'new' */
1979 8896100 : if (c != 'n') {
1980 8896100 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
1981 8896100 : if (ret < 0) {
1982 : /* LCOV_EXCL_START */
1983 : decoding_error(path, f);
1984 : os_abort();
1985 : /* LCOV_EXCL_STOP */
1986 : }
1987 : } else {
1988 : /* set the ZERO hash for deprecated NEW blocks */
1989 0 : hash_zero_set(block->hash);
1990 : }
1991 :
1992 : /* if the block contains a hash of past data */
1993 : /* and we are clearing such indeterminate hashes */
1994 8896100 : if (state->clear_past_hash
1995 2542004 : && block_has_past_hash(block)
1996 : ) {
1997 : /* set the hash value to INVALID */
1998 28006 : hash_invalid_set(block->hash);
1999 : }
2000 :
2001 : /* if we are disabling the copy optimization */
2002 : /* we want also to clear any already previously stored information */
2003 : /* in other sync commands */
2004 : /* note that this is required only in sync, and we detect */
2005 : /* this using the clear_past_hash flag */
2006 8896100 : if (state->clear_past_hash
2007 2542004 : && state->opt.force_nocopy
2008 14195 : && block_state_get(block) == BLOCK_STATE_REP
2009 : ) {
2010 : /* set the hash value to INVALID */
2011 1 : hash_invalid_set(block->hash);
2012 : /* convert from REP to CHG block */
2013 1 : block_state_set(block, BLOCK_STATE_CHG);
2014 : }
2015 :
2016 : /* if we want a full reallocation, marks block as invalid parity */
2017 : /* note that we do this after the force_nocopy option */
2018 : /* to avoid to mixup the two things */
2019 8896100 : if (state->opt.force_realloc
2020 27826 : && block_state_get(block) == BLOCK_STATE_BLK) {
2021 : /* convert from BLK to REP */
2022 27826 : block_state_set(block, BLOCK_STATE_REP);
2023 : }
2024 :
2025 : /* set the parity association */
2026 8896100 : fs_allocate(disk, v_pos, file, v_idx);
2027 :
2028 : /* go to the next block */
2029 8896100 : ++v_idx;
2030 8896100 : ++v_pos;
2031 8896100 : --v_count;
2032 : }
2033 : }
2034 :
2035 : /* stat */
2036 3611762 : ++count_file;
2037 123597 : } else if (c == 'i') {
2038 : /* "inf" command */
2039 : snapraid_info info;
2040 : uint32_t v_pos;
2041 : uint32_t v_oldest;
2042 :
2043 253 : ret = sgetb32(f, &v_oldest);
2044 253 : if (ret < 0) {
2045 : /* LCOV_EXCL_START */
2046 : decoding_error(path, f);
2047 : os_abort();
2048 : /* LCOV_EXCL_STOP */
2049 : }
2050 :
2051 253 : v_pos = 0;
2052 55893 : while (v_pos < blockmax) {
2053 : int bad;
2054 : int rehash;
2055 : int justsynced;
2056 : uint32_t t;
2057 : uint32_t flag;
2058 : uint32_t v_count;
2059 :
2060 55387 : ret = sgetb32(f, &v_count);
2061 55387 : if (ret < 0) {
2062 : /* LCOV_EXCL_START */
2063 : decoding_error(path, f);
2064 : os_abort();
2065 : /* LCOV_EXCL_STOP */
2066 : }
2067 :
2068 55387 : if (v_pos + v_count > blockmax) {
2069 : /* LCOV_EXCL_START */
2070 : decoding_error(path, f);
2071 : log_fatal("Internal inconsistency in info size %u/%u!\n", blockmax, v_pos + v_count);
2072 : os_abort();
2073 : /* LCOV_EXCL_STOP */
2074 : }
2075 :
2076 55387 : ret = sgetb32(f, &flag);
2077 55387 : if (ret < 0) {
2078 : /* LCOV_EXCL_START */
2079 : decoding_error(path, f);
2080 : os_abort();
2081 : /* LCOV_EXCL_STOP */
2082 : }
2083 :
2084 : /* if there is an info */
2085 55387 : if ((flag & 1) != 0) {
2086 : /* read the time */
2087 55372 : ret = sgetb32(f, &t);
2088 55372 : if (ret < 0) {
2089 : /* LCOV_EXCL_START */
2090 : decoding_error(path, f);
2091 : os_abort();
2092 : /* LCOV_EXCL_STOP */
2093 : }
2094 :
2095 : /* analyze the flags */
2096 55372 : bad = (flag & 2) != 0;
2097 55372 : rehash = (flag & 4) != 0;
2098 55372 : justsynced = (flag & 8) != 0;
2099 :
2100 55372 : if (rehash && state->prevhash == HASH_UNDEFINED) {
2101 : /* LCOV_EXCL_START */
2102 : decoding_error(path, f);
2103 : log_fatal("Internal inconsistency for missing previous checksum!\n");
2104 : os_abort();
2105 : /* LCOV_EXCL_STOP */
2106 : }
2107 :
2108 55372 : info = info_make(t + v_oldest, bad, rehash, justsynced);
2109 : } else {
2110 15 : info = 0;
2111 : }
2112 :
2113 1658697 : while (v_count) {
2114 : /* insert the info in the array */
2115 1547923 : info_set(&state->infoarr, v_pos, info);
2116 :
2117 : /* ensure that an info is present only for used positions */
2118 1547923 : if (fs_info_is_required(state, v_pos)) {
2119 1489158 : if (!info) {
2120 : /* LCOV_EXCL_START */
2121 : decoding_error(path, f);
2122 : log_fatal("Internal inconsistency for missing info!\n");
2123 : os_abort();
2124 : /* LCOV_EXCL_STOP */
2125 : }
2126 : } else {
2127 : /* extra info are accepted for backward compatibility */
2128 : /* they are discarded at the first write */
2129 : }
2130 :
2131 : /* go to next block */
2132 1547923 : ++v_pos;
2133 1547923 : --v_count;
2134 : }
2135 : }
2136 123344 : } else if (c == 'h') {
2137 : /* hole */
2138 : uint32_t v_pos;
2139 : struct snapraid_disk* disk;
2140 : uint32_t mapping;
2141 :
2142 1397 : ret = sgetb32(f, &mapping);
2143 1397 : if (ret < 0 || mapping >= mapping_max) {
2144 : /* LCOV_EXCL_START */
2145 : decoding_error(path, f);
2146 : log_fatal("Internal inconsistency in mapping index!\n");
2147 : os_abort();
2148 : /* LCOV_EXCL_STOP */
2149 : }
2150 1397 : disk = tommy_array_get(&disk_mapping, mapping);
2151 :
2152 1397 : v_pos = 0;
2153 4257 : while (v_pos < blockmax) {
2154 : uint32_t v_idx;
2155 : uint32_t v_count;
2156 : struct snapraid_file* deleted;
2157 :
2158 1463 : ret = sgetb32(f, &v_count);
2159 1463 : if (ret < 0) {
2160 : /* LCOV_EXCL_START */
2161 : decoding_error(path, f);
2162 : os_abort();
2163 : /* LCOV_EXCL_STOP */
2164 : }
2165 :
2166 1463 : if (v_pos + v_count > blockmax) {
2167 : /* LCOV_EXCL_START */
2168 : decoding_error(path, f);
2169 : log_fatal("Internal inconsistency in hole size %u/%u!\n", blockmax, v_pos + v_count);
2170 : os_abort();
2171 : /* LCOV_EXCL_STOP */
2172 : }
2173 :
2174 : /* get the sub-command */
2175 1463 : c = sgetc(f);
2176 :
2177 1463 : switch (c) {
2178 : case 'o' :
2179 : /* if it's a run of deleted blocks */
2180 :
2181 : /* allocate a fake deleted file */
2182 35 : deleted = file_alloc(state->block_size, "<deleted>", v_count * (data_off_t)state->block_size, 0, 0, 0, 0);
2183 :
2184 : /* mark the file as deleted */
2185 35 : file_flag_set(deleted, FILE_IS_DELETED);
2186 :
2187 : /* insert it in the list of deleted files */
2188 35 : tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted);
2189 :
2190 : /* process all blocks */
2191 35 : v_idx = 0;
2192 27956 : while (v_count) {
2193 27886 : struct snapraid_block* block = fs_file2block_get(deleted, v_idx);
2194 :
2195 : /* set the block as deleted */
2196 27886 : block_state_set(block, BLOCK_STATE_DELETED);
2197 :
2198 : /* read the hash */
2199 27886 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
2200 27886 : if (ret < 0) {
2201 : /* LCOV_EXCL_START */
2202 : decoding_error(path, f);
2203 : os_abort();
2204 : /* LCOV_EXCL_STOP */
2205 : }
2206 :
2207 : /* if we are clearing indeterminate hashes */
2208 27886 : if (state->clear_past_hash) {
2209 : /* set the hash value to INVALID */
2210 9432 : hash_invalid_set(block->hash);
2211 : }
2212 :
2213 : /* insert the block in the block array */
2214 27886 : fs_allocate(disk, v_pos, deleted, v_idx);
2215 :
2216 : /* go to next block */
2217 27886 : ++v_pos;
2218 27886 : ++v_idx;
2219 27886 : --v_count;
2220 : }
2221 35 : break;
2222 : case 'O' :
2223 : /* go to the next run */
2224 1428 : v_pos += v_count;
2225 1428 : break;
2226 : default :
2227 : /* LCOV_EXCL_START */
2228 : decoding_error(path, f);
2229 : log_fatal("Invalid hole type!\n");
2230 : os_abort();
2231 : /* LCOV_EXCL_STOP */
2232 : }
2233 : }
2234 121947 : } else if (c == 's') {
2235 : /* symlink */
2236 : char sub[PATH_MAX];
2237 : char linkto[PATH_MAX];
2238 : struct snapraid_link* slink;
2239 : struct snapraid_disk* disk;
2240 : uint32_t mapping;
2241 :
2242 116468 : ret = sgetb32(f, &mapping);
2243 116468 : if (ret < 0 || mapping >= mapping_max) {
2244 : /* LCOV_EXCL_START */
2245 : decoding_error(path, f);
2246 : log_fatal("Internal inconsistency in mapping index!\n");
2247 : os_abort();
2248 : /* LCOV_EXCL_STOP */
2249 : }
2250 116468 : disk = tommy_array_get(&disk_mapping, mapping);
2251 :
2252 116468 : ret = sgetbs(f, sub, sizeof(sub));
2253 116468 : if (ret < 0) {
2254 : /* LCOV_EXCL_START */
2255 : decoding_error(path, f);
2256 : os_abort();
2257 : /* LCOV_EXCL_STOP */
2258 : }
2259 :
2260 116468 : if (!*sub) {
2261 : /* LCOV_EXCL_START */
2262 : decoding_error(path, f);
2263 : log_fatal("Internal inconsistency for null symlink!\n");
2264 : os_abort();
2265 : /* LCOV_EXCL_STOP */
2266 : }
2267 :
2268 116468 : ret = sgetbs(f, linkto, sizeof(linkto));
2269 116468 : if (ret < 0) {
2270 : /* LCOV_EXCL_START */
2271 : decoding_error(path, f);
2272 : os_abort();
2273 : /* LCOV_EXCL_STOP */
2274 : }
2275 :
2276 : /* allocate the link as symbolic link */
2277 116468 : slink = link_alloc(sub, linkto, FILE_IS_SYMLINK);
2278 :
2279 : /* insert the link in the link containers */
2280 116468 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2281 116468 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2282 :
2283 : /* stat */
2284 116468 : ++count_symlink;
2285 5479 : } else if (c == 'a') {
2286 : /* hardlink */
2287 : char sub[PATH_MAX];
2288 : char linkto[PATH_MAX];
2289 : struct snapraid_link* slink;
2290 : struct snapraid_disk* disk;
2291 : uint32_t mapping;
2292 :
2293 597 : ret = sgetb32(f, &mapping);
2294 597 : if (ret < 0 || mapping >= mapping_max) {
2295 : /* LCOV_EXCL_START */
2296 : decoding_error(path, f);
2297 : log_fatal("Internal inconsistency in mapping index!\n");
2298 : os_abort();
2299 : /* LCOV_EXCL_STOP */
2300 : }
2301 597 : disk = tommy_array_get(&disk_mapping, mapping);
2302 :
2303 597 : ret = sgetbs(f, sub, sizeof(sub));
2304 597 : if (ret < 0) {
2305 : /* LCOV_EXCL_START */
2306 : decoding_error(path, f);
2307 : os_abort();
2308 : /* LCOV_EXCL_STOP */
2309 : }
2310 :
2311 597 : if (!*sub) {
2312 : /* LCOV_EXCL_START */
2313 : decoding_error(path, f);
2314 : log_fatal("Internal inconsistency for null hardlink!\n");
2315 : os_abort();
2316 : /* LCOV_EXCL_STOP */
2317 : }
2318 :
2319 597 : ret = sgetbs(f, linkto, sizeof(linkto));
2320 597 : if (ret < 0) {
2321 : /* LCOV_EXCL_START */
2322 : decoding_error(path, f);
2323 : os_abort();
2324 : /* LCOV_EXCL_STOP */
2325 : }
2326 :
2327 597 : if (!*linkto) {
2328 : /* LCOV_EXCL_START */
2329 : decoding_error(path, f);
2330 : log_fatal("Internal inconsistency for empty hardlink '%s'!\n", sub);
2331 : os_abort();
2332 : /* LCOV_EXCL_STOP */
2333 : }
2334 :
2335 : /* allocate the link as hard link */
2336 597 : slink = link_alloc(sub, linkto, FILE_IS_HARDLINK);
2337 :
2338 : /* insert the link in the link containers */
2339 597 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2340 597 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2341 :
2342 : /* stat */
2343 597 : ++count_hardlink;
2344 4882 : } else if (c == 'r') {
2345 : /* dir */
2346 : char sub[PATH_MAX];
2347 : struct snapraid_dir* dir;
2348 : struct snapraid_disk* disk;
2349 : uint32_t mapping;
2350 :
2351 790 : ret = sgetb32(f, &mapping);
2352 790 : if (ret < 0 || mapping >= mapping_max) {
2353 : /* LCOV_EXCL_START */
2354 : decoding_error(path, f);
2355 : log_fatal("Internal inconsistency in mapping index!\n");
2356 : os_abort();
2357 : /* LCOV_EXCL_STOP */
2358 : }
2359 790 : disk = tommy_array_get(&disk_mapping, mapping);
2360 :
2361 790 : ret = sgetbs(f, sub, sizeof(sub));
2362 790 : if (ret < 0) {
2363 : /* LCOV_EXCL_START */
2364 : decoding_error(path, f);
2365 : os_abort();
2366 : /* LCOV_EXCL_STOP */
2367 : }
2368 :
2369 790 : if (!*sub) {
2370 : /* LCOV_EXCL_START */
2371 : decoding_error(path, f);
2372 : log_fatal("Internal inconsistency for null dir!\n");
2373 : os_abort();
2374 : /* LCOV_EXCL_STOP */
2375 : }
2376 :
2377 : /* allocate the dir */
2378 790 : dir = dir_alloc(sub);
2379 :
2380 : /* insert the dir in the dir containers */
2381 790 : tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
2382 790 : tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
2383 :
2384 : /* stat */
2385 790 : ++count_dir;
2386 4092 : } else if (c == 'c') {
2387 : /* get the subcommand */
2388 254 : c = sgetc(f);
2389 :
2390 254 : switch (c) {
2391 : case 'u' :
2392 166 : state->hash = HASH_MURMUR3;
2393 166 : break;
2394 : case 'k' :
2395 88 : state->hash = HASH_SPOOKY2;
2396 88 : break;
2397 : default :
2398 : /* LCOV_EXCL_START */
2399 : decoding_error(path, f);
2400 : log_fatal("Invalid checksum!\n");
2401 : os_abort();
2402 : /* LCOV_EXCL_STOP */
2403 : }
2404 :
2405 : /* read the seed */
2406 254 : ret = sread(f, state->hashseed, HASH_MAX);
2407 254 : if (ret < 0) {
2408 : /* LCOV_EXCL_START */
2409 : decoding_error(path, f);
2410 : os_abort();
2411 : /* LCOV_EXCL_STOP */
2412 : }
2413 3838 : } else if (c == 'C') {
2414 : /* get the sub-command */
2415 4 : c = sgetc(f);
2416 :
2417 4 : switch (c) {
2418 : case 'u' :
2419 4 : state->prevhash = HASH_MURMUR3;
2420 4 : break;
2421 : case 'k' :
2422 0 : state->prevhash = HASH_SPOOKY2;
2423 0 : break;
2424 : default :
2425 : /* LCOV_EXCL_START */
2426 : decoding_error(path, f);
2427 : log_fatal("Invalid checksum!\n");
2428 : os_abort();
2429 : /* LCOV_EXCL_STOP */
2430 : }
2431 :
2432 : /* read the seed */
2433 4 : ret = sread(f, state->prevhashseed, HASH_MAX);
2434 4 : if (ret < 0) {
2435 : /* LCOV_EXCL_START */
2436 : decoding_error(path, f);
2437 : os_abort();
2438 : /* LCOV_EXCL_STOP */
2439 : }
2440 3834 : } else if (c == 'z') {
2441 : uint32_t block_size;
2442 :
2443 254 : ret = sgetb32(f, &block_size);
2444 254 : if (ret < 0) {
2445 : /* LCOV_EXCL_START */
2446 : decoding_error(path, f);
2447 : os_abort();
2448 : /* LCOV_EXCL_STOP */
2449 : }
2450 :
2451 254 : if (block_size == 0) {
2452 : /* LCOV_EXCL_START */
2453 : decoding_error(path, f);
2454 : log_fatal("Zero 'blocksize' specification in the content file!\n");
2455 : exit(EXIT_FAILURE);
2456 : /* LCOV_EXCL_STOP */
2457 : }
2458 :
2459 : /* without configuration, auto assign the block size */
2460 254 : if (state->no_conf) {
2461 1 : state->block_size = block_size;
2462 : }
2463 :
2464 254 : if (block_size != state->block_size) {
2465 : /* LCOV_EXCL_START */
2466 : decoding_error(path, f);
2467 : log_fatal("Mismatching 'blocksize' specification in the content file!\n");
2468 : log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", block_size / KIBI);
2469 : exit(EXIT_FAILURE);
2470 : /* LCOV_EXCL_STOP */
2471 : }
2472 3580 : } else if (c == 'y') {
2473 : uint32_t hash_size;
2474 :
2475 254 : ret = sgetb32(f, &hash_size);
2476 254 : if (ret < 0) {
2477 : /* LCOV_EXCL_START */
2478 : decoding_error(path, f);
2479 : os_abort();
2480 : /* LCOV_EXCL_STOP */
2481 : }
2482 :
2483 254 : if (hash_size < 2 || hash_size > HASH_MAX) {
2484 : /* LCOV_EXCL_START */
2485 : decoding_error(path, f);
2486 : log_fatal("Invalid 'hashsize' specification in the content file!\n");
2487 : exit(EXIT_FAILURE);
2488 : /* LCOV_EXCL_STOP */
2489 : }
2490 :
2491 : /* without configuration, auto assign the block size */
2492 254 : if (state->no_conf) {
2493 1 : BLOCK_HASH_SIZE = hash_size;
2494 : }
2495 :
2496 254 : if ((int)hash_size != BLOCK_HASH_SIZE) {
2497 : /* LCOV_EXCL_START */
2498 : decoding_error(path, f);
2499 : log_fatal("Mismatching 'hashsize' specification in the content file!\n");
2500 : log_fatal("Please restore the 'hashsize' value in the configuration file to '%u'\n", hash_size);
2501 : exit(EXIT_FAILURE);
2502 : /* LCOV_EXCL_STOP */
2503 : }
2504 3326 : } else if (c == 'x') {
2505 254 : ret = sgetb32(f, &blockmax);
2506 254 : if (ret < 0) {
2507 : /* LCOV_EXCL_START */
2508 : decoding_error(path, f);
2509 : os_abort();
2510 : /* LCOV_EXCL_STOP */
2511 : }
2512 4475 : } else if (c == 'm' || c == 'M') {
2513 : struct snapraid_map* map;
2514 : char uuid[UUID_MAX];
2515 : uint32_t v_pos;
2516 : uint32_t v_total_blocks;
2517 : uint32_t v_free_blocks;
2518 : struct snapraid_disk* disk;
2519 :
2520 1403 : ret = sgetbs(f, buffer, sizeof(buffer));
2521 1403 : if (ret < 0) {
2522 : /* LCOV_EXCL_START */
2523 : decoding_error(path, f);
2524 : os_abort();
2525 : /* LCOV_EXCL_STOP */
2526 : }
2527 :
2528 1403 : ret = sgetb32(f, &v_pos);
2529 1403 : if (ret < 0) {
2530 : /* LCOV_EXCL_START */
2531 : decoding_error(path, f);
2532 : os_abort();
2533 : /* LCOV_EXCL_STOP */
2534 : }
2535 :
2536 : /* from SnapRAID 7.0 the 'M' command includes the free space */
2537 1403 : if (c == 'M') {
2538 1403 : ret = sgetb32(f, &v_total_blocks);
2539 1403 : if (ret < 0) {
2540 : /* LCOV_EXCL_START */
2541 : decoding_error(path, f);
2542 : os_abort();
2543 : /* LCOV_EXCL_STOP */
2544 : }
2545 :
2546 1403 : ret = sgetb32(f, &v_free_blocks);
2547 1403 : if (ret < 0) {
2548 : /* LCOV_EXCL_START */
2549 : decoding_error(path, f);
2550 : os_abort();
2551 : /* LCOV_EXCL_STOP */
2552 : }
2553 : } else {
2554 0 : v_total_blocks = 0;
2555 0 : v_free_blocks = 0;
2556 : }
2557 :
2558 : /* read the uuid */
2559 1403 : ret = sgetbs(f, uuid, sizeof(uuid));
2560 1403 : if (ret < 0) {
2561 : /* LCOV_EXCL_START */
2562 : decoding_error(path, f);
2563 : os_abort();
2564 : /* LCOV_EXCL_STOP */
2565 : }
2566 :
2567 : /* find the disk */
2568 1403 : disk = find_disk_by_name(state, buffer);
2569 1403 : if (!disk) {
2570 : /* search by UUID if renamed */
2571 2 : disk = find_disk_by_uuid(state, uuid);
2572 2 : if (disk) {
2573 2 : log_fatal("WARNING! Renaming disk '%s' to '%s'\n", buffer, disk->name);
2574 :
2575 : /* write the new state with the new name */
2576 2 : state->need_write = 1;
2577 : }
2578 : }
2579 1403 : if (!disk) {
2580 : /* LCOV_EXCL_START */
2581 : decoding_error(path, f);
2582 : log_fatal("Disk '%s' with uuid '%s' not present in the configuration file!\n", buffer, uuid);
2583 : log_fatal("If you have removed it from the configuration file, please restore it\n");
2584 : /* if it's a command without UUID, it cannot autorename using UUID */
2585 : if (state->opt.skip_disk_access)
2586 : log_fatal("If you have renamed it, run 'sync' to update the new name\n");
2587 : exit(EXIT_FAILURE);
2588 : /* LCOV_EXCL_STOP */
2589 : }
2590 :
2591 1403 : map = map_alloc(disk->name, v_pos, v_total_blocks, v_free_blocks, uuid);
2592 :
2593 1403 : tommy_list_insert_tail(&state->maplist, &map->node, map);
2594 :
2595 : /* insert in the mapping vector */
2596 1403 : tommy_array_grow(&disk_mapping, mapping_max + 1);
2597 1403 : tommy_array_set(&disk_mapping, mapping_max, disk);
2598 1403 : ++mapping_max;
2599 1669 : } else if (c == 'P') {
2600 : /* from SnapRAID 7.0 the 'P' command includes the free space */
2601 : /* from SnapRAID 11.0 the 'P' command is deprecated by 'Q' */
2602 : char v_uuid[UUID_MAX];
2603 : uint32_t v_level;
2604 : uint32_t v_total_blocks;
2605 : uint32_t v_free_blocks;
2606 :
2607 0 : ret = sgetb32(f, &v_level);
2608 0 : if (ret < 0) {
2609 : /* LCOV_EXCL_START */
2610 : decoding_error(path, f);
2611 : os_abort();
2612 : /* LCOV_EXCL_STOP */
2613 : }
2614 :
2615 0 : ret = sgetb32(f, &v_total_blocks);
2616 0 : if (ret < 0) {
2617 : /* LCOV_EXCL_START */
2618 : decoding_error(path, f);
2619 : os_abort();
2620 : /* LCOV_EXCL_STOP */
2621 : }
2622 :
2623 0 : ret = sgetb32(f, &v_free_blocks);
2624 0 : if (ret < 0) {
2625 : /* LCOV_EXCL_START */
2626 : decoding_error(path, f);
2627 : os_abort();
2628 : /* LCOV_EXCL_STOP */
2629 : }
2630 :
2631 0 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2632 0 : if (ret < 0) {
2633 : /* LCOV_EXCL_START */
2634 : decoding_error(path, f);
2635 : os_abort();
2636 : /* LCOV_EXCL_STOP */
2637 : }
2638 :
2639 0 : if (v_level >= LEV_MAX) {
2640 : /* LCOV_EXCL_START */
2641 : decoding_error(path, f);
2642 : log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2643 : exit(EXIT_FAILURE);
2644 : /* LCOV_EXCL_STOP */
2645 : }
2646 :
2647 : /* auto configure if configuration is missing */
2648 0 : if (state->no_conf) {
2649 0 : if (v_level >= state->level)
2650 0 : state->level = v_level + 1;
2651 : }
2652 :
2653 : /* if we use this parity entry */
2654 0 : if (v_level < state->level) {
2655 : /* set the parity info */
2656 0 : state->parity[v_level].split_mac = 1;
2657 0 : pathcpy(state->parity[v_level].split_map[0].uuid, sizeof(state->parity[v_level].split_map[0].uuid), v_uuid);
2658 0 : state->parity[v_level].total_blocks = v_total_blocks;
2659 0 : state->parity[v_level].free_blocks = v_free_blocks;
2660 : }
2661 1669 : } else if (c == 'Q') {
2662 : /* from SnapRAID 11.0 the 'Q' command include size info and multi file support */
2663 : uint32_t v_level;
2664 : uint32_t v_total_blocks;
2665 : uint32_t v_free_blocks;
2666 : uint32_t v_split_mac;
2667 : unsigned s;
2668 :
2669 1415 : ret = sgetb32(f, &v_level);
2670 1415 : if (ret < 0) {
2671 : /* LCOV_EXCL_START */
2672 : decoding_error(path, f);
2673 : os_abort();
2674 : /* LCOV_EXCL_STOP */
2675 : }
2676 :
2677 1415 : ret = sgetb32(f, &v_total_blocks);
2678 1415 : if (ret < 0) {
2679 : /* LCOV_EXCL_START */
2680 : decoding_error(path, f);
2681 : os_abort();
2682 : /* LCOV_EXCL_STOP */
2683 : }
2684 :
2685 1415 : ret = sgetb32(f, &v_free_blocks);
2686 1415 : if (ret < 0) {
2687 : /* LCOV_EXCL_START */
2688 : decoding_error(path, f);
2689 : os_abort();
2690 : /* LCOV_EXCL_STOP */
2691 : }
2692 :
2693 1415 : ret = sgetb32(f, &v_split_mac);
2694 1415 : if (ret < 0) {
2695 : /* LCOV_EXCL_START */
2696 : decoding_error(path, f);
2697 : os_abort();
2698 : /* LCOV_EXCL_STOP */
2699 : }
2700 :
2701 1415 : if (v_level >= LEV_MAX) {
2702 : /* LCOV_EXCL_START */
2703 : decoding_error(path, f);
2704 : log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2705 : exit(EXIT_FAILURE);
2706 : /* LCOV_EXCL_STOP */
2707 : }
2708 :
2709 : /* auto configure if configuration is missing */
2710 1415 : if (state->no_conf) {
2711 6 : if (v_level >= state->level)
2712 5 : state->level = v_level + 1;
2713 6 : if (state->parity[v_level].split_mac < v_split_mac)
2714 6 : state->parity[v_level].split_mac = v_split_mac;
2715 : }
2716 :
2717 : /* if we use this parity entry */
2718 1415 : if (v_level < state->level) {
2719 : /* set the parity info */
2720 1156 : state->parity[v_level].total_blocks = v_total_blocks;
2721 1156 : state->parity[v_level].free_blocks = v_free_blocks;
2722 : }
2723 :
2724 7075 : for (s = 0; s < v_split_mac; ++s) {
2725 : char v_path[PATH_MAX];
2726 : char v_uuid[UUID_MAX];
2727 : uint64_t v_size;
2728 :
2729 5660 : ret = sgetbs(f, v_path, sizeof(v_path));
2730 5660 : if (ret < 0) {
2731 : /* LCOV_EXCL_START */
2732 : decoding_error(path, f);
2733 : os_abort();
2734 : /* LCOV_EXCL_STOP */
2735 : }
2736 :
2737 5660 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2738 5660 : if (ret < 0) {
2739 : /* LCOV_EXCL_START */
2740 : decoding_error(path, f);
2741 : os_abort();
2742 : /* LCOV_EXCL_STOP */
2743 : }
2744 :
2745 5660 : ret = sgetb64(f, &v_size);
2746 5660 : if (ret < 0) {
2747 : /* LCOV_EXCL_START */
2748 : decoding_error(path, f);
2749 : os_abort();
2750 : /* LCOV_EXCL_STOP */
2751 : }
2752 :
2753 : /* if we use this parity entry */
2754 5660 : if (v_level < state->level) {
2755 : /* if this split was removed from the configuration */
2756 4624 : if (s >= state->parity[v_level].split_mac) {
2757 : /* if the file is used, we really need it */
2758 0 : if (v_size != 0) {
2759 : /* LCOV_EXCL_START */
2760 : decoding_error(path, f);
2761 : log_fatal("Parity '%s' misses used file '%u'!\n", lev_config_name(v_level), s);
2762 : log_fatal("If you have removed it from the configuration file, please restore it\n");
2763 : exit(EXIT_FAILURE);
2764 : /* LCOV_EXCL_STOP */
2765 : }
2766 :
2767 : /* otherwise we can drop it */
2768 0 : log_fatal("WARNING! Dropping from '%s' unused split '%u'\n", lev_config_name(v_level), s);
2769 : } else {
2770 : /* we copy the path only if without configuration file */
2771 4624 : if (state->no_conf)
2772 24 : pathcpy(state->parity[v_level].split_map[s].path, sizeof(state->parity[v_level].split_map[s].path), v_path);
2773 :
2774 : /* set the split info */
2775 4624 : pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
2776 4624 : state->parity[v_level].split_map[s].size = v_size;
2777 :
2778 : /* log the info read from the content file */
2779 9248 : log_tag("content:%s:%u:%s:%s:%" PRIi64 "\n", lev_config_name(v_level), s,
2780 4624 : state->parity[v_level].split_map[s].path,
2781 4624 : state->parity[v_level].split_map[s].uuid,
2782 : state->parity[v_level].split_map[s].size);
2783 : }
2784 : }
2785 : }
2786 254 : } else if (c == 'N') {
2787 : uint32_t crc_stored;
2788 : uint32_t crc_computed;
2789 :
2790 : /* get the crc before reading it from the file */
2791 253 : crc_computed = scrc(f);
2792 :
2793 253 : ret = sgetble32(f, &crc_stored);
2794 253 : if (ret < 0) {
2795 : /* LCOV_EXCL_START */
2796 : /* here don't call decoding_error() because it's too late to get the crc */
2797 : log_fatal("Error reading the CRC in '%s' at offset %" PRIi64 "\n", path, stell(f));
2798 : log_fatal("This content file is damaged! Use an alternate copy.\n");
2799 : exit(EXIT_FAILURE);
2800 : /* LCOV_EXCL_STOP */
2801 : }
2802 :
2803 253 : if (crc_stored != crc_computed) {
2804 : /* LCOV_EXCL_START */
2805 : /* here don't call decoding_error() because it's too late to get the crc */
2806 : log_fatal("Mismatching CRC in '%s'\n", path);
2807 : log_fatal("This content file is damaged! Use an alternate copy.\n");
2808 : exit(EXIT_FAILURE);
2809 : /* LCOV_EXCL_STOP */
2810 : }
2811 :
2812 253 : crc_checked = 1;
2813 : } else {
2814 : /* LCOV_EXCL_START */
2815 : decoding_error(path, f);
2816 : log_fatal("Invalid command '%c'!\n", (char)c);
2817 : os_abort();
2818 : /* LCOV_EXCL_STOP */
2819 : }
2820 3735358 : }
2821 :
2822 253 : tommy_array_done(&disk_mapping);
2823 :
2824 253 : if (serror(f)) {
2825 : /* LCOV_EXCL_START */
2826 : log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
2827 : exit(EXIT_FAILURE);
2828 : /* LCOV_EXCL_STOP */
2829 : }
2830 :
2831 253 : if (!crc_checked) {
2832 : /* LCOV_EXCL_START */
2833 : log_fatal("Finished reading '%s' without finding the CRC\n", path);
2834 : log_fatal("This content file is truncated or damaged! Use an alternate copy.\n");
2835 : exit(EXIT_FAILURE);
2836 : /* LCOV_EXCL_STOP */
2837 : }
2838 :
2839 : /* check the file-system on all disks */
2840 253 : state_fscheck(state, "after read");
2841 :
2842 : /* check that the stored parity size matches the loaded state */
2843 253 : if (blockmax != parity_allocated_size(state)) {
2844 : /* LCOV_EXCL_START */
2845 : log_fatal("Internal inconsistency in parity size %u/%u in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f));
2846 : if (state->opt.skip_content_check) {
2847 : log_fatal("Overriding.\n");
2848 : blockmax = parity_allocated_size(state);
2849 : } else {
2850 : exit(EXIT_FAILURE);
2851 : }
2852 : /* LCOV_EXCL_STOP */
2853 : }
2854 :
2855 253 : msg_verbose("%8u files\n", count_file);
2856 253 : msg_verbose("%8u hardlinks\n", count_hardlink);
2857 253 : msg_verbose("%8u symlinks\n", count_symlink);
2858 253 : msg_verbose("%8u empty dirs\n", count_dir);
2859 253 : }
2860 :
2861 : struct state_write_thread_context {
2862 : struct snapraid_state* state;
2863 : #if HAVE_MT_WRITE
2864 : pthread_t thread;
2865 : #endif
2866 : /* input */
2867 : block_off_t blockmax;
2868 : time_t info_oldest;
2869 : int info_has_rehash;
2870 : STREAM* f;
2871 : /* output */
2872 : uint32_t crc;
2873 : unsigned count_file;
2874 : unsigned count_hardlink;
2875 : unsigned count_symlink;
2876 : unsigned count_dir;
2877 : };
2878 :
2879 151 : static void* state_write_thread(void* arg)
2880 : {
2881 151 : struct state_write_thread_context* context = arg;
2882 151 : struct snapraid_state* state = context->state;
2883 151 : block_off_t blockmax = context->blockmax;
2884 151 : time_t info_oldest = context->info_oldest;
2885 151 : int info_has_rehash = context->info_has_rehash;
2886 151 : STREAM* f = context->f;
2887 : uint32_t crc;
2888 : unsigned count_file;
2889 : unsigned count_hardlink;
2890 : unsigned count_symlink;
2891 : unsigned count_dir;
2892 : tommy_node* i;
2893 : block_off_t idx;
2894 : block_off_t begin;
2895 : unsigned l, s;
2896 : int version;
2897 :
2898 151 : count_file = 0;
2899 151 : count_hardlink = 0;
2900 151 : count_symlink = 0;
2901 151 : count_dir = 0;
2902 :
2903 : /* check what version to use */
2904 151 : version = 2;
2905 1012 : for (l = 0; l < state->level; ++l) {
2906 861 : if (state->parity[l].split_mac > 1)
2907 861 : version = 3;
2908 : }
2909 151 : if (BLOCK_HASH_SIZE != 16)
2910 0 : version = 3;
2911 :
2912 : /* write header */
2913 151 : if (version == 3)
2914 151 : swrite("SNAPCNT3\n\3\0\0", 12, f);
2915 : else
2916 0 : swrite("SNAPCNT2\n\3\0\0", 12, f);
2917 :
2918 : /* write block size and block max */
2919 151 : sputc('z', f);
2920 151 : sputb32(state->block_size, f);
2921 151 : sputc('x', f);
2922 151 : sputb32(blockmax, f);
2923 :
2924 : /* hash size */
2925 151 : if (version == 3) {
2926 151 : sputc('y', f);
2927 151 : sputb32(BLOCK_HASH_SIZE, f);
2928 : }
2929 :
2930 151 : if (serror(f)) {
2931 : /* LCOV_EXCL_START */
2932 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2933 : return context;
2934 : /* LCOV_EXCL_STOP */
2935 : }
2936 :
2937 151 : sputc('c', f);
2938 151 : if (state->hash == HASH_MURMUR3) {
2939 122 : sputc('u', f);
2940 29 : } else if (state->hash == HASH_SPOOKY2) {
2941 29 : sputc('k', f);
2942 : } else {
2943 : /* LCOV_EXCL_START */
2944 : log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f));
2945 : return context;
2946 : /* LCOV_EXCL_STOP */
2947 : }
2948 151 : swrite(state->hashseed, HASH_MAX, f);
2949 151 : if (serror(f)) {
2950 : /* LCOV_EXCL_START */
2951 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2952 : return context;
2953 : /* LCOV_EXCL_STOP */
2954 : }
2955 :
2956 : /* previous hash only present */
2957 151 : if (state->prevhash != HASH_UNDEFINED) {
2958 : /* if at least one rehash tag found, we have to save the previous hash */
2959 3 : if (info_has_rehash) {
2960 2 : sputc('C', f);
2961 2 : if (state->prevhash == HASH_MURMUR3) {
2962 2 : sputc('u', f);
2963 0 : } else if (state->prevhash == HASH_SPOOKY2) {
2964 0 : sputc('k', f);
2965 : } else {
2966 : /* LCOV_EXCL_START */
2967 : log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
2968 : return context;
2969 : /* LCOV_EXCL_STOP */
2970 : }
2971 2 : swrite(state->prevhashseed, HASH_MAX, f);
2972 2 : if (serror(f)) {
2973 : /* LCOV_EXCL_START */
2974 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2975 : return context;
2976 : /* LCOV_EXCL_STOP */
2977 : }
2978 : }
2979 : }
2980 :
2981 : /* for each map */
2982 1057 : for (i = state->maplist; i != 0; i = i->next) {
2983 906 : struct snapraid_map* map = i->data;
2984 : struct snapraid_disk* disk;
2985 :
2986 : /* find the disk for this mapping */
2987 906 : disk = find_disk_by_name(state, map->name);
2988 906 : if (!disk) {
2989 : /* LCOV_EXCL_START */
2990 : log_fatal("Internal inconsistency for unmapped disk '%s'\n", map->name);
2991 : return context;
2992 : /* LCOV_EXCL_STOP */
2993 : }
2994 :
2995 : /* save the mapping only if disk is mapped */
2996 906 : if (disk->mapping_idx != -1) {
2997 814 : sputc('M', f);
2998 814 : sputbs(map->name, f);
2999 814 : sputb32(map->position, f);
3000 814 : sputb32(map->total_blocks, f);
3001 814 : sputb32(map->free_blocks, f);
3002 814 : sputbs(map->uuid, f);
3003 814 : if (serror(f)) {
3004 : /* LCOV_EXCL_START */
3005 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3006 : return context;
3007 : /* LCOV_EXCL_STOP */
3008 : }
3009 : }
3010 : }
3011 :
3012 : /* for each parity */
3013 1012 : for (l = 0; l < state->level; ++l) {
3014 861 : if (version == 3) {
3015 861 : sputc('Q', f);
3016 861 : sputb32(l, f);
3017 861 : sputb32(state->parity[l].total_blocks, f);
3018 861 : sputb32(state->parity[l].free_blocks, f);
3019 861 : sputb32(state->parity[l].split_mac, f);
3020 4305 : for (s = 0; s < state->parity[l].split_mac; ++s) {
3021 3444 : sputbs(state->parity[l].split_map[s].path, f);
3022 3444 : sputbs(state->parity[l].split_map[s].uuid, f);
3023 3444 : sputb64(state->parity[l].split_map[s].size, f);
3024 : }
3025 : } else {
3026 0 : sputc('P', f);
3027 0 : sputb32(l, f);
3028 0 : sputb32(state->parity[l].total_blocks, f);
3029 0 : sputb32(state->parity[l].free_blocks, f);
3030 0 : sputbs(state->parity[l].split_map[0].uuid, f);
3031 : }
3032 861 : if (serror(f)) {
3033 : /* LCOV_EXCL_START */
3034 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3035 : return context;
3036 : /* LCOV_EXCL_STOP */
3037 : }
3038 : }
3039 :
3040 : /* for each disk */
3041 1057 : for (i = state->disklist; i != 0; i = i->next) {
3042 : tommy_node* j;
3043 906 : struct snapraid_disk* disk = i->data;
3044 :
3045 : /* if the disk is not mapped, skip it */
3046 906 : if (disk->mapping_idx < 0)
3047 92 : continue;
3048 :
3049 : /* for each file */
3050 1771698 : for (j = disk->filelist; j != 0; j = j->next) {
3051 1770884 : struct snapraid_file* file = j->data;
3052 : uint64_t size;
3053 : uint64_t mtime_sec;
3054 : int32_t mtime_nsec;
3055 : uint64_t inode;
3056 :
3057 1770884 : size = file->size;
3058 1770884 : mtime_sec = file->mtime_sec;
3059 1770884 : mtime_nsec = file->mtime_nsec;
3060 1770884 : inode = file->inode;
3061 :
3062 1770884 : sputc('f', f);
3063 1770884 : sputb32(disk->mapping_idx, f);
3064 1770884 : sputb64(size, f);
3065 1770884 : sputb64(mtime_sec, f);
3066 : /* encode STAT_NSEC_INVALID as 0 */
3067 1770884 : if (mtime_nsec == STAT_NSEC_INVALID)
3068 0 : sputb32(0, f);
3069 : else
3070 1770884 : sputb32(mtime_nsec + 1, f);
3071 1770884 : sputb64(inode, f);
3072 1770884 : sputbs(file->sub, f);
3073 1770884 : if (serror(f)) {
3074 : /* LCOV_EXCL_START */
3075 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3076 : return context;
3077 : /* LCOV_EXCL_STOP */
3078 : }
3079 :
3080 : /* for all the blocks of the file */
3081 1770884 : begin = 0;
3082 5322967 : while (begin < file->blockmax) {
3083 1781199 : unsigned v_state = block_state_get(fs_file2block_get(file, begin));
3084 1781199 : block_off_t v_pos = fs_file2par_get(disk, file, begin);
3085 : uint32_t v_count;
3086 :
3087 : block_off_t end;
3088 :
3089 : /* find the end of run of blocks */
3090 1781199 : end = begin + 1;
3091 6146339 : while (end < file->blockmax) {
3092 2596728 : if (v_state != block_state_get(fs_file2block_get(file, end)))
3093 12 : break;
3094 2596716 : if (v_pos + (end - begin) != fs_file2par_get(disk, file, end))
3095 12775 : break;
3096 2583941 : ++end;
3097 : }
3098 :
3099 1781199 : switch (v_state) {
3100 : case BLOCK_STATE_BLK :
3101 1678540 : sputc('b', f);
3102 1678540 : break;
3103 : case BLOCK_STATE_CHG :
3104 67741 : sputc('g', f);
3105 67741 : break;
3106 : case BLOCK_STATE_REP :
3107 34918 : sputc('p', f);
3108 34918 : break;
3109 : default :
3110 : /* LCOV_EXCL_START */
3111 : log_fatal("Internal inconsistency in state for block %u state %u\n", v_pos, v_state);
3112 : return context;
3113 : /* LCOV_EXCL_STOP */
3114 : }
3115 :
3116 1781199 : sputb32(v_pos, f);
3117 :
3118 1781199 : v_count = end - begin;
3119 1781199 : sputb32(v_count, f);
3120 :
3121 : /* write hashes */
3122 6146339 : for (idx = begin; idx < end; ++idx) {
3123 4365140 : struct snapraid_block* block = fs_file2block_get(file, idx);
3124 :
3125 4365140 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3126 : }
3127 :
3128 1781199 : if (serror(f)) {
3129 : /* LCOV_EXCL_START */
3130 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3131 : return context;
3132 : /* LCOV_EXCL_STOP */
3133 : }
3134 :
3135 : /* next begin position */
3136 1781199 : begin = end;
3137 : }
3138 :
3139 1770884 : ++count_file;
3140 : }
3141 :
3142 : /* for each link */
3143 58887 : for (j = disk->linklist; j != 0; j = j->next) {
3144 58073 : struct snapraid_link* slink = j->data;
3145 :
3146 58073 : switch (link_flag_get(slink, FILE_IS_LINK_MASK)) {
3147 : case FILE_IS_HARDLINK :
3148 321 : sputc('a', f);
3149 321 : ++count_hardlink;
3150 321 : break;
3151 : case FILE_IS_SYMLINK :
3152 57752 : sputc('s', f);
3153 57752 : ++count_symlink;
3154 57752 : break;
3155 : }
3156 :
3157 58073 : sputb32(disk->mapping_idx, f);
3158 58073 : sputbs(slink->sub, f);
3159 58073 : sputbs(slink->linkto, f);
3160 58073 : if (serror(f)) {
3161 : /* LCOV_EXCL_START */
3162 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3163 : return context;
3164 : /* LCOV_EXCL_STOP */
3165 : }
3166 : }
3167 :
3168 : /* for each dir */
3169 1241 : for (j = disk->dirlist; j != 0; j = j->next) {
3170 427 : struct snapraid_dir* dir = j->data;
3171 :
3172 427 : sputc('r', f);
3173 427 : sputb32(disk->mapping_idx, f);
3174 427 : sputbs(dir->sub, f);
3175 427 : if (serror(f)) {
3176 : /* LCOV_EXCL_START */
3177 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3178 : return context;
3179 : /* LCOV_EXCL_STOP */
3180 : }
3181 :
3182 427 : ++count_dir;
3183 : }
3184 :
3185 : /* deleted blocks of the disk */
3186 814 : sputc('h', f);
3187 814 : sputb32(disk->mapping_idx, f);
3188 814 : if (serror(f)) {
3189 : /* LCOV_EXCL_START */
3190 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3191 : return context;
3192 : /* LCOV_EXCL_STOP */
3193 : }
3194 814 : begin = 0;
3195 2516 : while (begin < blockmax) {
3196 : int is_deleted;
3197 : block_off_t end;
3198 :
3199 888 : is_deleted = fs_is_block_deleted(disk, begin);
3200 :
3201 : /* find the end of run of blocks */
3202 888 : end = begin + 1;
3203 4530435 : while (end < blockmax
3204 4528733 : && is_deleted == fs_is_block_deleted(disk, end)
3205 : ) {
3206 4528659 : ++end;
3207 : }
3208 :
3209 888 : sputb32(end - begin, f);
3210 :
3211 888 : if (is_deleted) {
3212 : /* write the run of deleted blocks with hash */
3213 47 : sputc('o', f);
3214 :
3215 : /* write all the hash */
3216 15961 : while (begin < end) {
3217 15867 : struct snapraid_block* block = fs_par2block_get(disk, begin);
3218 :
3219 15867 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3220 :
3221 15867 : ++begin;
3222 : }
3223 : } else {
3224 : /* write the run of blocks without hash */
3225 : /* they can be either used or empty blocks */
3226 841 : sputc('O', f);
3227 :
3228 : /* next begin position */
3229 841 : begin = end;
3230 : }
3231 :
3232 888 : if (serror(f)) {
3233 : /* LCOV_EXCL_START */
3234 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3235 : return context;
3236 : /* LCOV_EXCL_STOP */
3237 : }
3238 : }
3239 : }
3240 :
3241 : /* write the info for each block */
3242 151 : sputc('i', f);
3243 151 : sputb32(info_oldest, f);
3244 151 : begin = 0;
3245 19777 : while (begin < blockmax) {
3246 : snapraid_info info;
3247 : block_off_t end;
3248 : time_t t;
3249 : unsigned flag;
3250 :
3251 19475 : info = info_get(&state->infoarr, begin);
3252 :
3253 : /* find the end of run of blocks */
3254 19475 : end = begin + 1;
3255 775574 : while (end < blockmax
3256 755951 : && info == info_get(&state->infoarr, end)
3257 : ) {
3258 736624 : ++end;
3259 : }
3260 :
3261 19475 : sputb32(end - begin, f);
3262 :
3263 : /* if there is info */
3264 19475 : if (info) {
3265 : /* other flags */
3266 19452 : flag = 1; /* info is present */
3267 19452 : if (info_get_bad(info))
3268 1330 : flag |= 2;
3269 19452 : if (info_get_rehash(info))
3270 5212 : flag |= 4;
3271 19452 : if (info_get_justsynced(info))
3272 12082 : flag |= 8;
3273 19452 : sputb32(flag, f);
3274 :
3275 19452 : t = info_get_time(info) - info_oldest;
3276 19452 : sputb32(t, f);
3277 : } else {
3278 : /* write a special 0 flag to mark missing info */
3279 23 : sputb32(0, f);
3280 : }
3281 :
3282 19475 : if (serror(f)) {
3283 : /* LCOV_EXCL_START */
3284 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3285 : return context;
3286 : /* LCOV_EXCL_STOP */
3287 : }
3288 :
3289 : /* next begin position */
3290 19475 : begin = end;
3291 : }
3292 :
3293 151 : sputc('N', f);
3294 :
3295 : /* flush data written to the disk */
3296 151 : if (sflush(f)) {
3297 : /* LCOV_EXCL_START */
3298 : log_fatal("Error writing the content file '%s' (in flush before crc). %s.\n", serrorfile(f), strerror(errno));
3299 : return context;
3300 : /* LCOV_EXCL_STOP */
3301 : }
3302 :
3303 : /* get the file crc */
3304 151 : crc = scrc(f);
3305 :
3306 : /* compare the crc of the data written to file */
3307 : /* with the one of the data written to the stream */
3308 151 : if (crc != scrc_stream(f)) {
3309 : /* LCOV_EXCL_START */
3310 : log_fatal("CRC mismatch writing the content stream.\n");
3311 : log_fatal("DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
3312 : log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3313 : return context;
3314 : /* LCOV_EXCL_STOP */
3315 : }
3316 :
3317 151 : sputble32(crc, f);
3318 151 : if (serror(f)) {
3319 : /* LCOV_EXCL_START */
3320 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3321 : return context;
3322 : /* LCOV_EXCL_STOP */
3323 : }
3324 :
3325 : /* set output variables */
3326 151 : context->crc = crc;
3327 151 : context->count_file = count_file;
3328 151 : context->count_hardlink = count_hardlink;
3329 151 : context->count_symlink = count_symlink;
3330 151 : context->count_dir = count_dir;
3331 :
3332 151 : return 0;
3333 : }
3334 :
3335 151 : static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
3336 : {
3337 : #if HAVE_MT_WRITE
3338 : int fail;
3339 : int first;
3340 : #else
3341 : STREAM* f;
3342 : unsigned count_content;
3343 : unsigned k;
3344 : struct state_write_thread_context* context;
3345 : void* retval;
3346 : #endif
3347 : tommy_node* i;
3348 : block_off_t blockmax;
3349 : time_t info_oldest;
3350 : int info_has_rehash;
3351 : int mapping_idx;
3352 : block_off_t idx;
3353 : uint32_t crc;
3354 : unsigned count_file;
3355 : unsigned count_hardlink;
3356 : unsigned count_symlink;
3357 : unsigned count_dir;
3358 :
3359 : /* blocks of all array */
3360 151 : blockmax = parity_allocated_size(state);
3361 :
3362 : /* check the file-system on all disks */
3363 151 : state_fscheck(state, "before write");
3364 :
3365 : /* clear the info for unused blocks */
3366 : /* and get some other info */
3367 151 : info_oldest = 0; /* oldest time in info */
3368 151 : info_has_rehash = 0; /* if there is a rehash info */
3369 756250 : for (idx = 0; idx < blockmax; ++idx) {
3370 : /* if the position is used */
3371 756099 : if (fs_position_is_required(state, idx)) {
3372 756099 : snapraid_info info = info_get(&state->infoarr, idx);
3373 :
3374 : /* only if there is some info to store */
3375 756099 : if (info) {
3376 711614 : time_t info_time = info_get_time(info);
3377 :
3378 711614 : if (!info_oldest || info_time < info_oldest)
3379 154 : info_oldest = info_time;
3380 :
3381 711614 : if (info_get_rehash(info))
3382 13822 : info_has_rehash = 1;
3383 : }
3384 : } else {
3385 : /* clear any previous info */
3386 0 : info_set(&state->infoarr, idx, 0);
3387 :
3388 : /* and clear any deleted blocks */
3389 0 : fs_position_clear_deleted(state, idx);
3390 : }
3391 : }
3392 :
3393 : /* map disks */
3394 151 : mapping_idx = 0;
3395 1057 : for (i = state->maplist; i != 0; i = i->next) {
3396 906 : struct snapraid_map* map = i->data;
3397 : struct snapraid_disk* disk;
3398 :
3399 : /* find the disk for this mapping */
3400 906 : disk = find_disk_by_name(state, map->name);
3401 906 : if (!disk) {
3402 : /* LCOV_EXCL_START */
3403 : log_fatal("Internal inconsistency for unmapped disk '%s'\n", map->name);
3404 : os_abort();
3405 : /* LCOV_EXCL_STOP */
3406 : }
3407 :
3408 : /* save the mapping only for not empty disks */
3409 906 : if (!fs_is_empty(disk, blockmax)) {
3410 : /* assign the mapping index used to identify disks */
3411 814 : disk->mapping_idx = mapping_idx;
3412 814 : ++mapping_idx;
3413 : } else {
3414 : /* mark the disk as without mapping */
3415 92 : disk->mapping_idx = -1;
3416 : }
3417 : }
3418 :
3419 : #if HAVE_MT_WRITE
3420 : /* start all writing threads */
3421 : i = tommy_list_head(&state->contentlist);
3422 : while (i) {
3423 : struct snapraid_content* content = i->data;
3424 : struct state_write_thread_context* context;
3425 : char tmp[PATH_MAX];
3426 : STREAM* f;
3427 :
3428 : msg_progress("Saving state to %s...\n", content->content);
3429 :
3430 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3431 : f = sopen_write(tmp);
3432 : if (f == 0) {
3433 : /* LCOV_EXCL_START */
3434 : log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
3435 : exit(EXIT_FAILURE);
3436 : /* LCOV_EXCL_STOP */
3437 : }
3438 :
3439 : /* allocate the thread context */
3440 : context = malloc_nofail(sizeof(struct state_write_thread_context));
3441 : content->context = context;
3442 :
3443 : /* initialize */
3444 : context->state = state;
3445 : context->blockmax = blockmax;
3446 : context->info_oldest = info_oldest;
3447 : context->info_has_rehash = info_has_rehash;
3448 : context->f = f;
3449 :
3450 : thread_create(&context->thread, 0, state_write_thread, context);
3451 :
3452 : i = i->next;
3453 : }
3454 :
3455 : /* join all thread */
3456 : fail = 0;
3457 : first = 1;
3458 : crc = 0;
3459 : count_file = 0;
3460 : count_hardlink = 0;
3461 : count_symlink = 0;
3462 : count_dir = 0;
3463 : i = tommy_list_head(&state->contentlist);
3464 : while (i) {
3465 : struct snapraid_content* content = i->data;
3466 : struct state_write_thread_context* context = content->context;
3467 : void* retval;
3468 :
3469 : thread_join(context->thread, &retval);
3470 :
3471 : if (retval) {
3472 : /* LCOV_EXCL_START */
3473 : fail = 1;
3474 : /* LCOV_EXCL_STOP */
3475 : } else {
3476 : STREAM* f = context->f;
3477 :
3478 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3479 : /* than even in a system crash event we have one valid copy of the file. */
3480 : if (sflush(f) != 0) {
3481 : /* LCOV_EXCL_START */
3482 : log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3483 : exit(EXIT_FAILURE);
3484 : /* LCOV_EXCL_STOP */
3485 : }
3486 :
3487 : #if HAVE_FSYNC
3488 : if (ssync(f) != 0) {
3489 : /* LCOV_EXCL_START */
3490 : log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3491 : exit(EXIT_FAILURE);
3492 : /* LCOV_EXCL_STOP */
3493 : }
3494 : #endif
3495 :
3496 : if (sclose(f) != 0) {
3497 : /* LCOV_EXCL_START */
3498 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3499 : exit(EXIT_FAILURE);
3500 : /* LCOV_EXCL_STOP */
3501 : }
3502 :
3503 : if (first) {
3504 : first = 0;
3505 : crc = context->crc;
3506 : count_file = context->count_file;
3507 : count_hardlink = context->count_hardlink;
3508 : count_symlink = context->count_symlink;
3509 : count_dir = context->count_dir;
3510 : } else {
3511 : if (crc != context->crc) {
3512 : /* LCOV_EXCL_START */
3513 : log_fatal("Different CRCs writing content streams.\n");
3514 : log_fatal("DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
3515 : log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3516 : exit(EXIT_FAILURE);
3517 : /* LCOV_EXCL_STOP */
3518 : }
3519 : }
3520 : }
3521 :
3522 : free(context);
3523 :
3524 : i = i->next;
3525 : }
3526 :
3527 : /* abort on failure */
3528 : if (fail) {
3529 : /* LCOV_EXCL_START */
3530 : exit(EXIT_FAILURE);
3531 : /* LCOV_EXCL_STOP */
3532 : }
3533 : #else
3534 : /* count the content files */
3535 151 : count_content = 0;
3536 151 : i = tommy_list_head(&state->contentlist);
3537 1314 : while (i) {
3538 1012 : struct snapraid_content* content = i->data;
3539 1012 : msg_progress("Saving state to %s...\n", content->content);
3540 1012 : ++count_content;
3541 1012 : i = i->next;
3542 : }
3543 :
3544 : /* open all the content files */
3545 151 : f = sopen_multi_write(count_content);
3546 151 : if (!f) {
3547 : /* LCOV_EXCL_START */
3548 : log_fatal("Error opening the content files.\n");
3549 : exit(EXIT_FAILURE);
3550 : /* LCOV_EXCL_STOP */
3551 : }
3552 :
3553 151 : k = 0;
3554 151 : i = tommy_list_head(&state->contentlist);
3555 1314 : while (i) {
3556 1012 : struct snapraid_content* content = i->data;
3557 : char tmp[PATH_MAX];
3558 1012 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3559 1012 : if (sopen_multi_file(f, k, tmp) != 0) {
3560 : /* LCOV_EXCL_START */
3561 : log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
3562 : exit(EXIT_FAILURE);
3563 : /* LCOV_EXCL_STOP */
3564 : }
3565 1012 : ++k;
3566 1012 : i = i->next;
3567 : }
3568 :
3569 : /* allocate the thread context */
3570 151 : context = malloc_nofail(sizeof(struct state_write_thread_context));
3571 :
3572 : /* initialize */
3573 151 : context->state = state;
3574 151 : context->blockmax = blockmax;
3575 151 : context->info_oldest = info_oldest;
3576 151 : context->info_has_rehash = info_has_rehash;
3577 151 : context->f = f;
3578 :
3579 151 : retval = state_write_thread(context);
3580 :
3581 : /* abort on failure */
3582 151 : if (retval) {
3583 : /* LCOV_EXCL_START */
3584 : exit(EXIT_FAILURE);
3585 : /* LCOV_EXCL_STOP */
3586 : }
3587 :
3588 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3589 : /* than even in a system crash event we have one valid copy of the file. */
3590 151 : if (sflush(f) != 0) {
3591 : /* LCOV_EXCL_START */
3592 : log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3593 : exit(EXIT_FAILURE);
3594 : /* LCOV_EXCL_STOP */
3595 : }
3596 :
3597 : #if HAVE_FSYNC
3598 151 : if (ssync(f) != 0) {
3599 : /* LCOV_EXCL_START */
3600 : log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3601 : exit(EXIT_FAILURE);
3602 : /* LCOV_EXCL_STOP */
3603 : }
3604 : #endif
3605 :
3606 151 : if (sclose(f) != 0) {
3607 : /* LCOV_EXCL_START */
3608 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3609 : exit(EXIT_FAILURE);
3610 : /* LCOV_EXCL_STOP */
3611 : }
3612 :
3613 151 : crc = context->crc;
3614 151 : count_file = context->count_file;
3615 151 : count_hardlink = context->count_hardlink;
3616 151 : count_symlink = context->count_symlink;
3617 151 : count_dir = context->count_dir;
3618 :
3619 151 : free(context);
3620 : #endif
3621 :
3622 151 : msg_verbose("%8u files\n", count_file);
3623 151 : msg_verbose("%8u hardlinks\n", count_hardlink);
3624 151 : msg_verbose("%8u symlinks\n", count_symlink);
3625 151 : msg_verbose("%8u empty dirs\n", count_dir);
3626 :
3627 151 : *out_crc = crc;
3628 151 : }
3629 :
3630 261 : void state_read(struct snapraid_state* state)
3631 : {
3632 : STREAM* f;
3633 : char path[PATH_MAX];
3634 : struct stat st;
3635 : tommy_node* node;
3636 : int ret;
3637 : int c;
3638 :
3639 : /* iterate over all the available content files and load the first one present */
3640 261 : f = 0;
3641 261 : node = tommy_list_head(&state->contentlist);
3642 572 : while (node) {
3643 304 : struct snapraid_content* content = node->data;
3644 304 : pathcpy(path, sizeof(path), content->content);
3645 :
3646 304 : if (!state->no_conf) {
3647 303 : log_tag("content:%s\n", path);
3648 303 : log_flush();
3649 : }
3650 304 : msg_progress("Loading state from %s...\n", path);
3651 :
3652 304 : f = sopen_read(path);
3653 304 : if (f != 0) {
3654 : /* if opened stop the search */
3655 254 : break;
3656 : } else {
3657 : /* if it's real error of an existing file, abort */
3658 50 : if (errno != ENOENT) {
3659 : /* LCOV_EXCL_START */
3660 : log_fatal("Error opening the content file '%s'. %s.\n", path, strerror(errno));
3661 : exit(EXIT_FAILURE);
3662 : /* LCOV_EXCL_STOP */
3663 : }
3664 :
3665 : /* otherwise continue */
3666 50 : if (node->next) {
3667 43 : log_fatal("WARNING! Content file '%s' not found, trying with another copy...\n", path);
3668 :
3669 : /* ensure to rewrite all the content files */
3670 43 : state->need_write = 1;
3671 : }
3672 : }
3673 :
3674 : /* next content file */
3675 50 : node = node->next;
3676 : }
3677 :
3678 : /* if not found, assume empty */
3679 261 : if (!f) {
3680 7 : log_fatal("No content file found. Assuming empty.\n");
3681 :
3682 : /* create the initial mapping */
3683 7 : state_map(state);
3684 267 : return;
3685 : }
3686 :
3687 : /* get the stat of the content file */
3688 254 : ret = fstat(shandle(f), &st);
3689 254 : if (ret != 0) {
3690 : /* LCOV_EXCL_START */
3691 : log_fatal("Error stating the content file '%s'. %s.\n", path, strerror(errno));
3692 : exit(EXIT_FAILURE);
3693 : /* LCOV_EXCL_STOP */
3694 : }
3695 :
3696 : /* go further to check other content files */
3697 1955 : while (node) {
3698 : char other_path[PATH_MAX];
3699 : struct stat other_st;
3700 1447 : struct snapraid_content* content = node->data;
3701 1447 : pathcpy(other_path, sizeof(other_path), content->content);
3702 :
3703 1447 : ret = stat(other_path, &other_st);
3704 1447 : if (ret != 0) {
3705 : /* allow missing content files, but not any other kind of error */
3706 1 : if (errno != ENOENT) {
3707 : /* LCOV_EXCL_START */
3708 : log_fatal("Error stating the content file '%s'. %s.\n", other_path, strerror(errno));
3709 : exit(EXIT_FAILURE);
3710 : /* LCOV_EXCL_STOP */
3711 : }
3712 :
3713 : /* ensure to rewrite all the content files */
3714 1 : state->need_write = 1;
3715 : } else {
3716 : /* if the size is different */
3717 1446 : if (other_st.st_size != st.st_size) {
3718 39 : log_fatal("WARNING! Content files '%s' and '%s' have a different size!\n", path, other_path);
3719 39 : log_fatal("Likely one of the two is broken!\n");
3720 :
3721 : /* ensure to rewrite all the content files */
3722 39 : state->need_write = 1;
3723 : }
3724 : }
3725 :
3726 : /* next content file */
3727 1447 : node = node->next;
3728 : }
3729 :
3730 : /* start with a undefined default. */
3731 : /* it's for compatibility with version 1.0 where MD5 was implicit. */
3732 254 : state->hash = HASH_UNDEFINED;
3733 :
3734 : /* start with a zero seed, it was the default in old versions */
3735 254 : memset(state->hashseed, 0, HASH_MAX);
3736 :
3737 : /* previous hash, start with an undefined value */
3738 254 : state->prevhash = HASH_UNDEFINED;
3739 :
3740 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
3741 :
3742 : /* get the first char to detect the file type */
3743 254 : c = sgetc(f);
3744 254 : sungetc(c, f);
3745 :
3746 : /* guess the file type from the first char */
3747 254 : if (c == 'S') {
3748 254 : state_read_content(state, path, f);
3749 : } else {
3750 : /* LCOV_EXCL_START */
3751 : log_fatal("From SnapRAID v9.0 the text content file is not supported anymore.\n");
3752 : log_fatal("You have first to upgrade to SnapRAID v8.1 to convert it to binary format.\n");
3753 : exit(EXIT_FAILURE);
3754 : /* LCOV_EXCL_STOP */
3755 : }
3756 :
3757 253 : sclose(f);
3758 :
3759 253 : if (state->hash == HASH_UNDEFINED) {
3760 : /* LCOV_EXCL_START */
3761 : log_fatal("The checksum to use is not specified.\n");
3762 : log_fatal("This happens because you are likely upgrading from SnapRAID 1.0.\n");
3763 : log_fatal("To use a new SnapRAID you must restart from scratch,\n");
3764 : log_fatal("deleting all the content and parity files.\n");
3765 : exit(EXIT_FAILURE);
3766 : /* LCOV_EXCL_STOP */
3767 : }
3768 :
3769 : /* update the mapping */
3770 253 : state_map(state);
3771 :
3772 253 : state_content_check(state, path);
3773 :
3774 : /* mark that we read the content file, and it passed all the checks */
3775 253 : state->checked_read = 1;
3776 : }
3777 :
3778 : struct state_verify_thread_context {
3779 : struct snapraid_state* state;
3780 : struct snapraid_content* content;
3781 : #if HAVE_MT_VERIFY
3782 : pthread_t thread;
3783 : #else
3784 : void* retval;
3785 : #endif
3786 : /* input */
3787 : uint32_t crc;
3788 : STREAM* f;
3789 : };
3790 :
3791 1012 : static void* state_verify_thread(void* arg)
3792 : {
3793 1012 : struct state_verify_thread_context* context = arg;
3794 1012 : STREAM* f = context->f;
3795 : unsigned char buf[4];
3796 : uint32_t crc_stored;
3797 : uint32_t crc_computed;
3798 :
3799 1012 : if (sdeplete(f, buf) != 0) {
3800 : /* LCOV_EXCL_START */
3801 : log_fatal("Error flushing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3802 : return context;
3803 : /* LCOV_EXCL_STOP */
3804 : }
3805 :
3806 : /* get the stored crc from the last four bytes */
3807 1012 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
3808 :
3809 1012 : if (crc_stored != context->crc) {
3810 : /* LCOV_EXCL_START */
3811 : log_fatal("DANGER! Wrong stored CRC in '%s'\n", serrorfile(f));
3812 : return context;
3813 : /* LCOV_EXCL_STOP */
3814 : }
3815 :
3816 : /* get the computed crc */
3817 1012 : crc_computed = scrc(f);
3818 :
3819 : /* adjust the stored crc to include itself */
3820 1012 : crc_stored = crc32c(crc_stored, buf, 4);
3821 :
3822 1012 : if (crc_computed != crc_stored) {
3823 : /* LCOV_EXCL_START */
3824 : log_fatal("DANGER! Wrong file CRC in '%s'\n", serrorfile(f));
3825 : return context;
3826 : /* LCOV_EXCL_STOP */
3827 : }
3828 :
3829 1012 : return 0;
3830 : }
3831 :
3832 151 : static void state_verify_content(struct snapraid_state* state, uint32_t crc)
3833 : {
3834 : tommy_node* i;
3835 : int fail;
3836 :
3837 : /* start all reading threads */
3838 151 : i = tommy_list_head(&state->contentlist);
3839 1314 : while (i) {
3840 1012 : struct snapraid_content* content = i->data;
3841 : struct state_verify_thread_context* context;
3842 : char tmp[PATH_MAX];
3843 : STREAM* f;
3844 :
3845 1012 : msg_progress("Verifying %s...\n", content->content);
3846 :
3847 1012 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3848 :
3849 1012 : f = sopen_read(tmp);
3850 1012 : if (f == 0) {
3851 : /* LCOV_EXCL_START */
3852 : log_fatal("Error reopening the content file '%s'. %s.\n", tmp, strerror(errno));
3853 : exit(EXIT_FAILURE);
3854 : /* LCOV_EXCL_STOP */
3855 : }
3856 :
3857 : /* allocate the thread context */
3858 1012 : context = malloc_nofail(sizeof(struct state_verify_thread_context));
3859 1012 : content->context = context;
3860 :
3861 : /* initialize */
3862 1012 : context->state = state;
3863 1012 : context->content = content;
3864 1012 : context->crc = crc;
3865 1012 : context->f = f;
3866 :
3867 : #if HAVE_MT_VERIFY
3868 1012 : thread_create(&context->thread, 0, state_verify_thread, context);
3869 : #else
3870 : context->retval = state_verify_thread(context);
3871 : #endif
3872 :
3873 1012 : i = i->next;
3874 : }
3875 :
3876 : /* join all thread */
3877 151 : fail = 0;
3878 151 : i = tommy_list_head(&state->contentlist);
3879 1314 : while (i) {
3880 1012 : struct snapraid_content* content = i->data;
3881 1012 : struct state_verify_thread_context* context = content->context;
3882 : void* retval;
3883 :
3884 : #if HAVE_MT_VERIFY
3885 1012 : thread_join(context->thread, &retval);
3886 : #else
3887 : retval = context->retval;
3888 : #endif
3889 1012 : if (retval) {
3890 : /* LCOV_EXCL_START */
3891 : fail = 1;
3892 : /* LCOV_EXCL_STOP */
3893 : } else {
3894 1012 : STREAM* f = context->f;
3895 :
3896 1012 : if (sclose(f) != 0) {
3897 : /* LCOV_EXCL_START */
3898 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3899 : exit(EXIT_FAILURE);
3900 : /* LCOV_EXCL_STOP */
3901 : }
3902 : }
3903 :
3904 1012 : free(context);
3905 :
3906 1012 : i = i->next;
3907 : }
3908 :
3909 : /* abort on failure */
3910 151 : if (fail) {
3911 : /* LCOV_EXCL_START */
3912 : exit(EXIT_FAILURE);
3913 : /* LCOV_EXCL_STOP */
3914 : }
3915 151 : }
3916 :
3917 151 : static void state_rename_content(struct snapraid_state* state)
3918 : {
3919 : tommy_node* i;
3920 :
3921 151 : i = tommy_list_head(&state->contentlist);
3922 1314 : while (i) {
3923 1012 : struct snapraid_content* content = i->data;
3924 : char tmp[PATH_MAX];
3925 :
3926 : /* now renames the just written copy with the correct name */
3927 1012 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3928 1012 : if (rename(tmp, content->content) != 0) {
3929 : /* LCOV_EXCL_START */
3930 : log_fatal("Error renaming the content file '%s' to '%s' in rename(). %s.\n", tmp, content->content, strerror(errno));
3931 : exit(EXIT_FAILURE);
3932 : /* LCOV_EXCL_STOP */
3933 : }
3934 :
3935 1012 : i = i->next;
3936 : }
3937 151 : }
3938 :
3939 151 : void state_write(struct snapraid_state* state)
3940 : {
3941 : uint32_t crc;
3942 :
3943 : /* write all the content files */
3944 151 : state_write_content(state, &crc);
3945 :
3946 : /* verify the just written files */
3947 151 : state_verify_content(state, crc);
3948 :
3949 : /* rename the new files, over the old ones */
3950 151 : state_rename_content(state);
3951 :
3952 151 : state->need_write = 0; /* no write needed anymore */
3953 151 : state->checked_read = 0; /* what we wrote is not checked in read */
3954 151 : }
3955 :
3956 122 : void state_skip(struct snapraid_state* state)
3957 : {
3958 : tommy_node* i;
3959 :
3960 : /* for each disk */
3961 850 : for (i = state->disklist; i != 0; i = i->next) {
3962 : tommy_node* j;
3963 728 : struct snapraid_disk* disk = i->data;
3964 :
3965 728 : if (!disk->skip_access)
3966 727 : continue;
3967 :
3968 : /* for each file */
3969 1885 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
3970 1884 : struct snapraid_file* file = j->data;
3971 1884 : file_flag_set(file, FILE_IS_EXCLUDED);
3972 : }
3973 :
3974 : /* for each link */
3975 57 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
3976 56 : struct snapraid_link* slink = j->data;
3977 56 : link_flag_set(slink, FILE_IS_EXCLUDED);
3978 : }
3979 :
3980 : /* for each dir */
3981 1 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
3982 0 : struct snapraid_dir* dir = j->data;
3983 0 : dir_flag_set(dir, FILE_IS_EXCLUDED);
3984 : }
3985 : }
3986 122 : }
3987 :
3988 122 : void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing, int filter_error)
3989 : {
3990 : tommy_node* i;
3991 : unsigned l;
3992 :
3993 : /* if no filter, include all */
3994 122 : if (!filter_missing && !filter_error && tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk))
3995 235 : return;
3996 :
3997 9 : msg_progress("Filtering...\n");
3998 :
3999 17 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
4000 8 : struct snapraid_filter* filter = i->data;
4001 8 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_disk ? "//" : "");
4002 : }
4003 12 : for (i = tommy_list_head(filterlist_file); i != 0; i = i->next) {
4004 3 : struct snapraid_filter* filter = i->data;
4005 3 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_dir ? "/" : "");
4006 : }
4007 9 : if (filter_missing)
4008 2 : msg_verbose("\t<missing>\n");
4009 9 : if (filter_error)
4010 2 : msg_verbose("\t<error>\n");
4011 :
4012 : /* for each disk */
4013 63 : for (i = state->disklist; i != 0; i = i->next) {
4014 : tommy_node* j;
4015 54 : struct snapraid_disk* disk = i->data;
4016 :
4017 : /* if we filter for presence, we have to access the disk, so better to print something */
4018 54 : if (filter_missing)
4019 12 : msg_progress("Scanning disk %s...\n", disk->name);
4020 :
4021 : /* for each file */
4022 133944 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
4023 133890 : struct snapraid_file* file = j->data;
4024 :
4025 133890 : if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0
4026 80982 : || filter_path(filterlist_file, 0, disk->name, file->sub) != 0
4027 78138 : || filter_existence(filter_missing, disk->dir, file->sub) != 0
4028 38646 : || filter_correctness(filter_error, &state->infoarr, disk, file) != 0
4029 : ) {
4030 109490 : file_flag_set(file, FILE_IS_EXCLUDED);
4031 : }
4032 : }
4033 :
4034 : /* for each link */
4035 4414 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
4036 4360 : struct snapraid_link* slink = j->data;
4037 :
4038 4360 : if (filter_path(filterlist_disk, 0, disk->name, slink->sub) != 0
4039 2641 : || filter_path(filterlist_file, 0, disk->name, slink->sub) != 0
4040 2536 : || filter_existence(filter_missing, disk->dir, slink->sub) != 0
4041 : ) {
4042 3221 : link_flag_set(slink, FILE_IS_EXCLUDED);
4043 : }
4044 : }
4045 :
4046 : /* for each empty dir */
4047 82 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
4048 28 : struct snapraid_dir* dir = j->data;
4049 :
4050 28 : if (filter_emptydir(filterlist_disk, 0, disk->name, dir->sub) != 0
4051 17 : || filter_emptydir(filterlist_file, 0, disk->name, dir->sub) != 0
4052 17 : || filter_existence(filter_missing, disk->dir, dir->sub) != 0
4053 : ) {
4054 19 : dir_flag_set(dir, FILE_IS_EXCLUDED);
4055 : }
4056 : }
4057 : }
4058 :
4059 : /* if we are filtering by disk */
4060 9 : if (!tommy_list_empty(filterlist_disk)) {
4061 : /* for each parity disk */
4062 20 : for (l = 0; l < state->level; ++l) {
4063 : /* check if the parity is excluded by name */
4064 16 : if (filter_path(filterlist_disk, 0, lev_config_name(l), 0) != 0) {
4065 : /* excluded the parity from further operation */
4066 11 : state->parity[l].is_excluded = 1;
4067 : }
4068 : }
4069 : } else {
4070 : /* if we are filtering by file, exclude all parity */
4071 5 : if (filter_missing || !tommy_list_empty(filterlist_file)) {
4072 : /* for each parity disk */
4073 16 : for (l = 0; l < state->level; ++l) {
4074 13 : state->parity[l].is_excluded = 1;
4075 : }
4076 : }
4077 : }
4078 : }
4079 :
4080 221 : int state_progress_begin(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, block_off_t countmax)
4081 : {
4082 : time_t now;
4083 :
4084 221 : if (state->opt.gui) {
4085 5 : log_tag("run:begin:%u:%u:%u\n", blockstart, blockmax, countmax);
4086 5 : log_flush();
4087 : }
4088 :
4089 221 : now = time(0);
4090 :
4091 221 : state->progress_whole_start = now;
4092 :
4093 221 : state->progress_tick = 0;
4094 221 : state->progress_ptr = 0;
4095 221 : state->progress_wasted = 0;
4096 :
4097 : /* stop if requested */
4098 221 : if (global_interrupt) {
4099 : /* LCOV_EXCL_START */
4100 : if (!state->opt.gui) {
4101 : msg_status("Not starting for interruption\n");
4102 : }
4103 : log_tag("sigint:0: SIGINT received\n");
4104 : log_flush();
4105 : return 0;
4106 : /* LCOV_EXCL_STOP */
4107 : }
4108 :
4109 221 : return 1;
4110 : }
4111 :
4112 221 : void state_progress_end(struct snapraid_state* state, block_off_t countpos, block_off_t countmax, data_off_t countsize)
4113 : {
4114 221 : if (state->opt.gui) {
4115 5 : log_tag("run:end\n");
4116 5 : log_flush();
4117 216 : } else if (countmax == 0) {
4118 19 : msg_status("Nothing to do\n");
4119 : } else {
4120 : time_t now;
4121 : time_t elapsed;
4122 :
4123 197 : unsigned countsize_MB = (countsize + MEGA - 1) / MEGA;
4124 :
4125 197 : now = time(0);
4126 :
4127 197 : elapsed = now - state->progress_whole_start - state->progress_wasted;
4128 :
4129 197 : msg_bar("%u%% completed, %u MB accessed", countpos * 100 / countmax, countsize_MB);
4130 197 : if (elapsed >= 60)
4131 0 : msg_bar(" in %u:%02u", (unsigned)(elapsed / 3600), (unsigned)((elapsed % 3600) / 60));
4132 197 : msg_bar("\n");
4133 197 : msg_flush();
4134 : }
4135 221 : }
4136 :
4137 1 : void state_progress_stop(struct snapraid_state* state)
4138 : {
4139 : time_t now;
4140 :
4141 1 : now = time(0);
4142 :
4143 1 : if (!state->opt.gui) {
4144 1 : msg_bar("\n");
4145 1 : msg_flush();
4146 : }
4147 :
4148 1 : state->progress_interruption = now;
4149 1 : }
4150 :
4151 1 : void state_progress_restart(struct snapraid_state* state)
4152 : {
4153 : time_t now;
4154 :
4155 1 : now = time(0);
4156 :
4157 : /* reset the progress counter */
4158 1 : state->progress_tick = 0;
4159 1 : state->progress_ptr = 0;
4160 :
4161 1 : if (now >= state->progress_interruption) /* avoid degenerated cases when the clock is manually adjusted */
4162 1 : state->progress_wasted += now - state->progress_interruption;
4163 1 : }
4164 :
4165 : #define PROGRESS_CLEAR " "
4166 :
4167 : /**
4168 : * Set the latest tick data in the progress vector.
4169 : */
4170 532 : static void state_progress_latest(struct snapraid_state* state)
4171 : {
4172 : tommy_node* i;
4173 : unsigned l;
4174 :
4175 532 : state->progress_tick_misc[state->progress_ptr] = state->tick_misc;
4176 532 : state->progress_tick_sched[state->progress_ptr] = state->tick_sched;
4177 532 : state->progress_tick_raid[state->progress_ptr] = state->tick_raid;
4178 532 : state->progress_tick_hash[state->progress_ptr] = state->tick_hash;
4179 532 : state->progress_tick_io[state->progress_ptr] = state->tick_io;
4180 3716 : for (i = state->disklist; i != 0; i = i->next) {
4181 3184 : struct snapraid_disk* disk = i->data;
4182 3184 : disk->progress_tick[state->progress_ptr] = disk->tick;
4183 : }
4184 2946 : for (l = 0; l < state->level; ++l)
4185 2414 : state->parity[l].progress_tick[state->progress_ptr] = state->parity[l].tick;
4186 532 : }
4187 :
4188 : /**
4189 : * Number of columns
4190 : */
4191 : #define GRAPH_COLUMN 68
4192 :
4193 : /**
4194 : * Get the reference value, 0 if index is PROGRESS_MAX
4195 : */
4196 : #define ref(map, index) (index < PROGRESS_MAX ? map[index] : 0)
4197 :
4198 2 : static void state_progress_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest)
4199 : {
4200 : uint64_t v;
4201 : uint64_t tick_total;
4202 : unsigned bar;
4203 : tommy_node* i;
4204 : unsigned l;
4205 : size_t pad;
4206 :
4207 2 : tick_total = 0;
4208 :
4209 2 : tick_total += state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4210 2 : tick_total += state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4211 2 : tick_total += state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4212 2 : tick_total += state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4213 2 : tick_total += state->progress_tick_io[current] - ref(state->progress_tick_io, oldest);
4214 2 : if (!tick_total)
4215 2 : return;
4216 :
4217 2 : pad = 4;
4218 14 : for (i = state->disklist; i != 0; i = i->next) {
4219 12 : struct snapraid_disk* disk = i->data;
4220 12 : size_t len = strlen(disk->name);
4221 12 : if (pad < len)
4222 2 : pad = len;
4223 : }
4224 9 : for (l = 0; l < state->level; ++l) {
4225 7 : size_t len = strlen(lev_config_name(l));
4226 7 : if (pad < len)
4227 3 : pad = len;
4228 : }
4229 :
4230 : /* extra space */
4231 2 : pad += 1;
4232 :
4233 2 : if (pad + 30 < GRAPH_COLUMN)
4234 2 : bar = GRAPH_COLUMN - pad;
4235 : else
4236 0 : bar = 30;
4237 :
4238 2 : if (io) {
4239 1 : const char* legend = "cached blocks (instant, more is better)";
4240 :
4241 : /* refresh the cached blocks info */
4242 1 : io_refresh(io);
4243 :
4244 1 : printf("\n");
4245 :
4246 7 : for (i = state->disklist; i != 0; i = i->next) {
4247 6 : struct snapraid_disk* disk = i->data;
4248 6 : v = disk->cached;
4249 6 : printr(disk->name, pad);
4250 6 : printf("%4" PRIu64 " | ", v);
4251 6 : printc('o', v * bar / io->io_max);
4252 6 : printf("\n");
4253 : }
4254 :
4255 2 : for (l = 0; l < state->level; ++l) {
4256 1 : v = state->parity[l].cached;
4257 1 : printr(lev_config_name(l), pad);
4258 1 : printf("%4" PRIu64 " | ", v);
4259 1 : printc('o', v * bar / io->io_max);
4260 1 : printf("\n");
4261 : }
4262 :
4263 1 : printc(' ', pad);
4264 1 : printf(" |_");
4265 1 : printc('_', bar);
4266 1 : printf("\n");
4267 :
4268 1 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4269 1 : printf("%s", legend);
4270 1 : printf("\n");
4271 : }
4272 :
4273 2 : printf("\n");
4274 :
4275 14 : for (i = state->disklist; i != 0; i = i->next) {
4276 12 : struct snapraid_disk* disk = i->data;
4277 12 : v = disk->progress_tick[current] - ref(disk->progress_tick, oldest);
4278 12 : printr(disk->name, pad);
4279 12 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4280 12 : printc('*', v * bar / tick_total);
4281 12 : printf("\n");
4282 : }
4283 :
4284 9 : for (l = 0; l < state->level; ++l) {
4285 7 : v = state->parity[l].progress_tick[current] - ref(state->parity[l].progress_tick, oldest);
4286 7 : printr(lev_config_name(l), pad);
4287 7 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4288 7 : printc('*', v * bar / tick_total);
4289 7 : printf("\n");
4290 : }
4291 :
4292 2 : v = state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4293 2 : printr("raid", pad);
4294 2 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4295 2 : printc('*', v * bar / tick_total);
4296 2 : printf("\n");
4297 :
4298 2 : v = state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4299 2 : printr("hash", pad);
4300 2 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4301 2 : printc('*', v * bar / tick_total);
4302 2 : printf("\n");
4303 :
4304 2 : v = state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4305 2 : printr("sched", pad);
4306 2 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4307 2 : printc('*', v * bar / tick_total);
4308 2 : printf("\n");
4309 :
4310 2 : v = state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4311 2 : printr("misc", pad);
4312 2 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4313 2 : printc('*', v * bar / tick_total);
4314 2 : printf("\n");
4315 :
4316 2 : printc(' ', pad);
4317 2 : printf(" |_");
4318 2 : printc('_', bar);
4319 2 : printf("\n");
4320 :
4321 2 : if (oldest == PROGRESS_MAX) {
4322 1 : const char* legend = "wait time (total, less is better)";
4323 1 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4324 1 : printf("%s", legend);
4325 1 : printf("\n");
4326 : } else {
4327 1 : const char* legend_d = "wait time (last %d secs, less is better)";
4328 1 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend_d) / 2);
4329 1 : printf(legend_d, PROGRESS_MAX);
4330 1 : printf("\n");
4331 : }
4332 :
4333 2 : printf("\n");
4334 : }
4335 :
4336 1073436 : int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize)
4337 : {
4338 : time_t now;
4339 : int pred;
4340 :
4341 1073436 : now = time(0);
4342 :
4343 : /* previous position */
4344 1073436 : pred = state->progress_ptr + PROGRESS_MAX - 1;
4345 1073436 : if (pred >= PROGRESS_MAX)
4346 1073237 : pred -= PROGRESS_MAX;
4347 :
4348 : /* if the previous measure is different */
4349 1073436 : if (state->progress_tick == 0
4350 1073237 : || state->progress_time[pred] != now
4351 : ) {
4352 : time_t elapsed;
4353 436 : unsigned out_perc = 0;
4354 436 : unsigned out_speed = 0;
4355 436 : unsigned out_cpu = 0;
4356 436 : unsigned out_eta = 0;
4357 436 : int out_computed = 0;
4358 :
4359 : /* store the new measure */
4360 436 : state->progress_time[state->progress_ptr] = now;
4361 436 : state->progress_pos[state->progress_ptr] = countpos;
4362 436 : state->progress_size[state->progress_ptr] = countsize;
4363 436 : state_progress_latest(state);
4364 :
4365 436 : elapsed = now - state->progress_whole_start - state->progress_wasted;
4366 :
4367 : /* completion percentage */
4368 436 : if (countmax)
4369 436 : out_perc = countpos * 100 / countmax;
4370 :
4371 : /* if we have at least 5 measures */
4372 436 : if (state->progress_tick >= 5
4373 : /* or if we are running in test mode, with at least one measure */
4374 397 : || (state->opt.force_progress && state->progress_tick >= 1)
4375 : ) {
4376 : int oldest;
4377 : int past;
4378 : time_t delta_time;
4379 : block_off_t delta_pos;
4380 : data_off_t delta_size;
4381 : uint64_t tick_cpu;
4382 : uint64_t tick_total;
4383 : uint64_t delta_tick_cpu;
4384 : uint64_t delta_tick_total;
4385 : uint64_t oldest_tick_cpu;
4386 : uint64_t oldest_tick_total;
4387 :
4388 : /* number of past measures */
4389 237 : past = state->progress_tick;
4390 :
4391 : /* drop the oldest ones, to promptly */
4392 : /* skip the startup phase */
4393 237 : past -= past / 5;
4394 :
4395 : /* check how much we can go in the past */
4396 237 : if (past >= PROGRESS_MAX - 1) {
4397 : /* the vector is filled, so we are already in position */
4398 : /* to get the possible oldest one */
4399 0 : oldest = state->progress_ptr + 1;
4400 : } else {
4401 : /* go backward the number of positions selected */
4402 237 : oldest = state->progress_ptr + PROGRESS_MAX - past;
4403 : }
4404 237 : if (oldest >= PROGRESS_MAX)
4405 237 : oldest -= PROGRESS_MAX;
4406 :
4407 474 : tick_cpu = state->progress_tick_misc[state->progress_ptr]
4408 237 : + state->progress_tick_sched[state->progress_ptr]
4409 237 : + state->progress_tick_raid[state->progress_ptr]
4410 237 : + state->progress_tick_hash[state->progress_ptr];
4411 237 : tick_total = tick_cpu + state->progress_tick_io[state->progress_ptr];
4412 :
4413 474 : oldest_tick_cpu = state->progress_tick_misc[oldest]
4414 237 : + state->progress_tick_sched[oldest]
4415 237 : + state->progress_tick_raid[oldest]
4416 237 : + state->progress_tick_hash[oldest];
4417 237 : oldest_tick_total = oldest_tick_cpu + state->progress_tick_io[oldest];
4418 :
4419 237 : delta_time = now - state->progress_time[oldest];
4420 237 : delta_pos = countpos - state->progress_pos[oldest];
4421 237 : delta_size = countsize - state->progress_size[oldest];
4422 237 : delta_tick_cpu = tick_cpu - oldest_tick_cpu;
4423 237 : delta_tick_total = tick_total - oldest_tick_total;
4424 :
4425 : /* estimate the speed in MB/s */
4426 237 : if (delta_time != 0)
4427 237 : out_speed = (unsigned)(delta_size / MEGA / delta_time);
4428 :
4429 : /* estimate the cpu usage percentage */
4430 237 : if (delta_tick_total != 0)
4431 95 : out_cpu = (unsigned)(delta_tick_cpu * 100U / delta_tick_total);
4432 :
4433 : /* estimate the remaining time in minutes */
4434 237 : if (delta_pos != 0)
4435 237 : out_eta = (countmax - countpos) * delta_time / (60 * delta_pos);
4436 :
4437 237 : if (state->opt.force_stats) {
4438 1 : os_clear();
4439 1 : state_progress_graph(state, io, state->progress_ptr, oldest);
4440 : }
4441 :
4442 : /* we have the output value */
4443 237 : out_computed = 1;
4444 : }
4445 :
4446 436 : if (state->opt.gui) {
4447 7 : log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_speed, out_cpu, (uint64_t)elapsed);
4448 7 : log_flush();
4449 : } else {
4450 429 : msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA));
4451 429 : if (out_computed) {
4452 235 : msg_bar(", %u MB/s", out_speed);
4453 235 : msg_bar(", CPU %u%%", out_cpu);
4454 235 : msg_bar(", %u:%02u ETA", out_eta / 60, out_eta % 60);
4455 : }
4456 429 : msg_bar("%s\r", PROGRESS_CLEAR);
4457 429 : msg_flush();
4458 : }
4459 :
4460 : /* next position to fill */
4461 436 : ++state->progress_ptr;
4462 436 : if (state->progress_ptr >= PROGRESS_MAX)
4463 0 : state->progress_ptr -= PROGRESS_MAX;
4464 :
4465 : /* one more measure */
4466 436 : ++state->progress_tick;
4467 : }
4468 :
4469 : /* stop if requested */
4470 1073436 : if (global_interrupt) {
4471 : /* LCOV_EXCL_START */
4472 : if (!state->opt.gui) {
4473 : log_fatal("\n");
4474 : log_fatal("Stopping for interruption at block %u\n", blockpos);
4475 : }
4476 : log_tag("sigint:%u: SIGINT received\n", blockpos);
4477 : log_flush();
4478 : return 1;
4479 : /* LCOV_EXCL_STOP */
4480 : }
4481 :
4482 1073436 : return 0;
4483 : }
4484 :
4485 104 : void state_usage_waste(struct snapraid_state* state)
4486 : {
4487 104 : uint64_t now = tick();
4488 :
4489 104 : state->tick_last = now;
4490 104 : }
4491 :
4492 1063244 : void state_usage_misc(struct snapraid_state* state)
4493 : {
4494 1063244 : uint64_t now = tick();
4495 1063244 : uint64_t delta = now - state->tick_last;
4496 :
4497 : /* increment the time spent in computations */
4498 1063244 : state->tick_misc += delta;
4499 :
4500 1063244 : state->tick_last = now;
4501 1063244 : }
4502 :
4503 149863 : void state_usage_sched(struct snapraid_state* state)
4504 : {
4505 149863 : uint64_t now = tick();
4506 149863 : uint64_t delta = now - state->tick_last;
4507 :
4508 : /* increment the time spent in computations */
4509 149863 : state->tick_sched += delta;
4510 :
4511 149863 : state->tick_last = now;
4512 149863 : }
4513 :
4514 140402 : void state_usage_raid(struct snapraid_state* state)
4515 : {
4516 140402 : uint64_t now = tick();
4517 140402 : uint64_t delta = now - state->tick_last;
4518 :
4519 : /* increment the time spent in computations */
4520 140402 : state->tick_raid += delta;
4521 :
4522 140402 : state->tick_last = now;
4523 140402 : }
4524 :
4525 776953 : void state_usage_hash(struct snapraid_state* state)
4526 : {
4527 776953 : uint64_t now = tick();
4528 776953 : uint64_t delta = now - state->tick_last;
4529 :
4530 : /* increment the time spent in computations */
4531 776953 : state->tick_hash += delta;
4532 :
4533 776953 : state->tick_last = now;
4534 776953 : }
4535 :
4536 912519 : void state_usage_disk(struct snapraid_state* state, struct snapraid_handle* handle_map, unsigned* waiting_map, unsigned waiting_mac)
4537 : {
4538 912519 : uint64_t now = tick();
4539 912519 : uint64_t delta = now - state->tick_last;
4540 : unsigned i;
4541 :
4542 : /* increment the time spent in the data disks */
4543 957179 : for (i = 0; i < waiting_mac; ++i) {
4544 44660 : struct snapraid_disk* disk = handle_map[waiting_map[i]].disk;
4545 :
4546 44660 : if (!disk)
4547 10 : continue;
4548 :
4549 44650 : disk->tick += delta;
4550 : }
4551 912519 : state->tick_io += delta;
4552 :
4553 912519 : state->tick_last = now;
4554 912519 : }
4555 :
4556 774185 : void state_usage_parity(struct snapraid_state* state, unsigned* waiting_map, unsigned waiting_mac)
4557 : {
4558 774185 : uint64_t now = tick();
4559 774185 : uint64_t delta = now - state->tick_last;
4560 : unsigned i;
4561 :
4562 : /* increment the time spent in the parity disk */
4563 854258 : for (i = 0; i < waiting_mac; ++i)
4564 80073 : state->parity[waiting_map[i]].tick += delta;
4565 774185 : state->tick_io += delta;
4566 :
4567 774185 : state->tick_last = now;
4568 774185 : }
4569 :
4570 96 : void state_usage_print(struct snapraid_state* state)
4571 : {
4572 : /* set the latest data */
4573 96 : state_progress_latest(state);
4574 :
4575 96 : if (msg_level < MSG_PROGRESS)
4576 191 : return;
4577 :
4578 : /* print a graph for it */
4579 1 : state_progress_graph(state, 0, state->progress_ptr, PROGRESS_MAX);
4580 : }
4581 :
4582 499 : void state_fscheck(struct snapraid_state* state, const char* ope)
4583 : {
4584 : tommy_node* i;
4585 :
4586 : /* check the file-system on all disks */
4587 3487 : for (i = state->disklist; i != 0; i = i->next) {
4588 2988 : struct snapraid_disk* disk = i->data;
4589 :
4590 2988 : if (fs_check(disk) != 0) {
4591 : /* LCOV_EXCL_START */
4592 : log_fatal("Internal inconsistency in file-system for disk '%s' %s\n", disk->name, ope);
4593 : os_abort();
4594 : /* LCOV_EXCL_STOP */
4595 : }
4596 : }
4597 499 : }
4598 :
4599 1 : void generate_configuration(const char* path)
4600 : {
4601 : struct snapraid_state state;
4602 : struct snapraid_content* content;
4603 : char esc_buffer[ESC_MAX];
4604 : unsigned l, s;
4605 : tommy_node* j;
4606 :
4607 1 : state_init(&state);
4608 :
4609 : /* mark that we are without a configuration file */
4610 1 : state.no_conf = 1;
4611 :
4612 : /* create the dummy content entry */
4613 1 : content = content_alloc(path, -1);
4614 :
4615 : /* adds the content entry */
4616 1 : tommy_list_insert_tail(&state.contentlist, &content->node, content);
4617 :
4618 : /* read the content file */
4619 1 : state_read(&state);
4620 :
4621 : /* output a dummy configuration file */
4622 1 : printf("# Configuration file generated from %s\n", path);
4623 1 : printf("\n");
4624 1 : printf("# Use this blocksize\n");
4625 1 : printf("blocksize %u\n", state.block_size / KIBI);
4626 1 : printf("\n");
4627 1 : printf("# Use this hashsize\n");
4628 1 : printf("hashsize %u\n", BLOCK_HASH_SIZE);
4629 1 : printf("\n");
4630 7 : for (l = 0; l < state.level; ++l) {
4631 6 : printf("# Set the correct path for the %s files\n", lev_name(l));
4632 6 : printf("# You had %u of them:\n", state.parity[l].split_mac);
4633 30 : for (s = 0; s < state.parity[l].split_mac; ++s) {
4634 24 : printf("# %u:\n", s);
4635 24 : printf("# PATH:");
4636 24 : if (state.parity[l].split_map[s].path[0])
4637 24 : printf("%s", state.parity[l].split_map[s].path);
4638 : else
4639 0 : printf("?");
4640 24 : printf("\n");
4641 24 : printf("# SIZE:");
4642 24 : if (state.parity[l].split_map[s].size != PARITY_SIZE_INVALID)
4643 24 : printf("%" PRIu64, state.parity[l].split_map[s].size);
4644 : else
4645 0 : printf("?");
4646 24 : printf("\n");
4647 24 : printf("# UUID:");
4648 24 : if (state.parity[l].split_map[s].uuid[0])
4649 24 : printf("%s", state.parity[l].split_map[s].uuid);
4650 : else
4651 0 : printf("?");
4652 24 : printf("\n");
4653 24 : printf("#\n");
4654 : }
4655 6 : printf("%s ENTER_HERE_THE_PARITY_FILES_COMMA_SEPARATED\n", lev_config_name(l));
4656 6 : printf("\n");
4657 : }
4658 1 : printf("# Add any other content file\n");
4659 1 : printf("content %s\n", path);
4660 1 : printf("\n");
4661 7 : for (j = state.maplist; j; j = j->next) {
4662 6 : struct snapraid_map* map = j->data;
4663 : struct snapraid_disk* disk;
4664 6 : printf("# Set the correct dir for disk '%s'\n", map->name);
4665 6 : if (map->uuid[0])
4666 6 : printf("# Disk '%s' is the one with id '%s'\n", map->name, map->uuid);
4667 6 : disk = find_disk_by_name(&state, map->name);
4668 6 : if (disk && disk->filelist) {
4669 6 : struct snapraid_file* file = disk->filelist->data;
4670 6 : if (file) {
4671 6 : printf("# and containing: %s\n", fmt_poll(disk, file->sub, esc_buffer));
4672 : }
4673 : }
4674 6 : printf("data %s ENTER_HERE_THE_DIR\n", map->name);
4675 6 : printf("\n");
4676 : }
4677 :
4678 1 : state_done(&state);
4679 1 : }
4680 :
|