Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "support.h"
21 : #include "util.h"
22 : #include "elem.h"
23 : #include "import.h"
24 : #include "search.h"
25 : #include "state.h"
26 : #include "parity.h"
27 : #include "handle.h"
28 : #include "raid/raid.h"
29 : #include "raid/combo.h"
30 :
31 : /****************************************************************************/
32 : /* check */
33 :
34 : /**
35 : * A block that failed the hash check, or that was deleted.
36 : */
37 : struct failed_struct {
38 : /**
39 : * If we know for sure that the block is garbage or missing
40 : * and it needs to be recovered and rewritten to the disk.
41 : */
42 : int is_bad;
43 :
44 : /**
45 : * If that we have recovered may be not updated data,
46 : * an old version, or just garbage.
47 : *
48 : * Essentially, it means that we are not sure what we have recovered
49 : * is really correct. It's just our best guess.
50 : *
51 : * These "recovered" block are also written to the disk if the block is marked as ::is_bad.
52 : * But these files are marked also as FILE_IS_DAMAGED, and then renamed to .unrecoverable.
53 : *
54 : * Note that this could happen only for CHG blocks.
55 : */
56 : int is_outofdate;
57 :
58 : unsigned index; /**< Index of the failed block. */
59 : struct snapraid_block* block; /**< The failed block, or BLOCK_DELETED for a deleted block */
60 : struct snapraid_handle* handle; /**< The file containing the failed block, or 0 for a deleted block */
61 : };
62 :
63 : /**
64 : * Checks if a block hash matches the specified buffer.
65 : * Return ==0 if equal
66 : */
67 463419 : static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_block* block, unsigned char* buffer, unsigned char* buffer_zero)
68 : {
69 : unsigned char hash[HASH_SIZE];
70 : unsigned size;
71 :
72 463419 : size = block_file_size(block, state->block_size);
73 :
74 : /* now compute the hash of the valid part */
75 463419 : if (rehash) {
76 22265 : memhash(state->prevhash, state->prevhashseed, hash, buffer, size);
77 : } else {
78 441154 : memhash(state->hash, state->hashseed, hash, buffer, size);
79 : }
80 :
81 : /* compare the hash */
82 463419 : if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
83 13902 : return -1;
84 : }
85 :
86 : /* compare to the end of the block */
87 449517 : if (size < state->block_size) {
88 185818 : if (memcmp(buffer + size, buffer_zero + size, state->block_size - size) != 0) {
89 21 : return -1;
90 : }
91 : }
92 :
93 449496 : return 0;
94 : }
95 :
96 : /**
97 : * Checks if the hash of all the failed block we are expecting to recover are now matching.
98 : */
99 204860 : static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void* buffer_zero)
100 : {
101 : unsigned j;
102 : int hash_checked;
103 :
104 204860 : hash_checked = 0; /* keep track if we check at least one block */
105 :
106 : /* check if the recovered blocks are OK */
107 658878 : for (j = 0; j < failed_count; ++j) {
108 : /* if we are expected to recover this block */
109 467941 : if (!failed[failed_map[j]].is_outofdate
110 : /* if the block has a hash to check */
111 463419 : && block_has_updated_hash(failed[failed_map[j]].block)
112 : ) {
113 : /* if a hash doesn't match, fail the check */
114 463419 : if (blockcmp(state, rehash, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0) {
115 13923 : log_tag("hash_error: Hash mismatch on entry %u\n", failed_map[j]);
116 13923 : return 0;
117 : }
118 :
119 449496 : hash_checked = 1;
120 : }
121 : }
122 :
123 : /* if nothing checked, we reject it */
124 : /* note that we are excluding this case at upper level */
125 : /* but checking again doesn't hurt */
126 190937 : if (!hash_checked) {
127 : /* LCOV_EXCL_START */
128 : return 0;
129 : /* LCOV_EXCL_STOP */
130 : }
131 :
132 : /* if we checked something, and no block failed the check */
133 : /* recompute all the redundancy information */
134 190937 : raid_gen(diskmax, state->level, state->block_size, buffer);
135 190937 : return 1;
136 : }
137 :
138 : /**
139 : * Checks if specified parity is now matching with a recomputed one.
140 : */
141 8416 : static int is_parity_matching(struct snapraid_state* state, unsigned diskmax, unsigned i, void** buffer, void** buffer_recov)
142 : {
143 : /* recompute parity, note that we don't need parity over i */
144 8416 : raid_gen(diskmax, i + 1, state->block_size, buffer);
145 :
146 : /* if the recovered parity block matches */
147 8416 : if (memcmp(buffer[diskmax + i], buffer_recov[i], state->block_size) == 0) {
148 : /* recompute all the redundancy information */
149 8416 : raid_gen(diskmax, state->level, state->block_size, buffer);
150 8416 : return 1;
151 : }
152 :
153 0 : return 0;
154 : }
155 :
156 : /**
157 : * Repair errors.
158 : * Returns <0 if failure for missing strategy, >0 if data is wrong and we cannot rebuild correctly, 0 on success.
159 : * If success, the parity are computed in the buffer variable.
160 : */
161 256105 : static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void** buffer_recov, void* buffer_zero)
162 : {
163 : unsigned i, n;
164 : int error;
165 : int has_hash;
166 : int id[LEV_MAX];
167 : int ip[LEV_MAX];
168 :
169 : /* no fix required, already checked at higher level, but just to be sure */
170 256105 : if (failed_count == 0) {
171 : /* LCOV_EXCL_START */
172 : /* recompute only the parity */
173 : raid_gen(diskmax, state->level, state->block_size, buffer);
174 : return 0;
175 : /* LCOV_EXCL_STOP */
176 : }
177 :
178 256105 : n = state->level;
179 256105 : error = 0;
180 :
181 : /* setup vector of failed disk indexes */
182 898662 : for (i = 0; i < failed_count; ++i)
183 642557 : id[i] = failed[failed_map[i]].index;
184 :
185 : /* check if there is at least a failed block that can be checked for correctness using the hash */
186 : /* if there isn't, we have to sacrifice a parity block to check that the result is correct */
187 256105 : has_hash = 0;
188 898662 : for (i = 0; i < failed_count; ++i) {
189 : /* if we are expected to recover this block */
190 642557 : if (!failed[failed_map[i]].is_outofdate
191 : /* if the block has a hash to check */
192 633513 : && block_has_updated_hash(failed[failed_map[i]].block)
193 : )
194 625097 : has_hash = 1;
195 : }
196 :
197 : /* if we don't have a hash, but we have an extra parity */
198 : /* (strictly-less failures than number of parities) */
199 256105 : if (!has_hash && failed_count < n) {
200 : /* number of parity to use, one more to check the recovering */
201 8416 : unsigned r = failed_count + 1;
202 :
203 : /* all combinations (r of n) parities */
204 8416 : combination_first(r, n, ip);
205 : do {
206 : /* if a parity is missing, do nothing */
207 25248 : for (i = 0; i < r; ++i) {
208 16832 : if (buffer_recov[ip[i]] == 0)
209 0 : break;
210 : }
211 8416 : if (i != r)
212 0 : continue;
213 :
214 : /* copy the parities to use, one less because the last is used for checking */
215 16832 : for (i = 0; i < r - 1; ++i)
216 8416 : memcpy(buffer[diskmax + ip[i]], buffer_recov[ip[i]], state->block_size);
217 :
218 : /* recover using one less parity, the ip[r-1] one */
219 8416 : raid_data(r - 1, id, ip, diskmax, state->block_size, buffer);
220 :
221 : /* use the remaining ip[r-1] parity to check the result */
222 8416 : if (is_parity_matching(state, diskmax, ip[r - 1], buffer, buffer_recov))
223 8416 : return 0;
224 :
225 : /* log */
226 0 : log_tag("parity_error:%u:", pos);
227 0 : for (i = 0; i < r; ++i) {
228 0 : if (i != 0)
229 0 : log_tag("/");
230 0 : log_tag("%s", lev_config_name(ip[i]));
231 : }
232 0 : log_tag(":parity: Parity mismatch\n");
233 0 : ++error;
234 0 : } while (combination_next(r, n, ip));
235 : }
236 :
237 : /* if we have a hash, and enough parities */
238 : /* (less-or-equal failures than number of parities) */
239 247689 : if (has_hash && failed_count <= n) {
240 : /* number of parities to use equal at the number of failures */
241 200100 : unsigned r = failed_count;
242 :
243 : /* all combinations (r of n) parities */
244 200100 : combination_first(r, n, ip);
245 : do {
246 : /* if a parity is missing, do nothing */
247 677450 : for (i = 0; i < r; ++i) {
248 472590 : if (buffer_recov[ip[i]] == 0)
249 4649 : break;
250 : }
251 209509 : if (i != r)
252 4649 : continue;
253 :
254 : /* copy the parities to use */
255 672801 : for (i = 0; i < r; ++i)
256 467941 : memcpy(buffer[diskmax + ip[i]], buffer_recov[ip[i]], state->block_size);
257 :
258 : /* recover */
259 204860 : raid_data(r, id, ip, diskmax, state->block_size, buffer);
260 :
261 : /* use the hash to check the result */
262 204860 : if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero))
263 190937 : return 0;
264 :
265 : /* log */
266 13923 : log_tag("parity_error:%u:", pos);
267 27846 : for (i = 0; i < r; ++i) {
268 13923 : if (i != 0)
269 0 : log_tag("/");
270 13923 : log_tag("%s", lev_config_name(ip[i]));
271 : }
272 13923 : log_tag(":hash: Hash mismatch\n");
273 13923 : ++error;
274 37144 : } while (combination_next(r, n, ip));
275 : }
276 :
277 : /* return the number of failed attempts, or -1 if no strategy */
278 56752 : if (error)
279 9163 : return error;
280 :
281 47589 : log_tag("strategy_error:%u: No strategy to recover from %u failures with %u parity %s hash\n",
282 : pos, failed_count, n, has_hash ? "with" : "without");
283 47589 : return -1;
284 : }
285 :
286 869093 : static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void** buffer_recov, void* buffer_zero)
287 : {
288 : int ret;
289 : int error;
290 : unsigned j;
291 : int n;
292 : int something_to_recover;
293 : int something_unsynced;
294 :
295 869093 : error = 0;
296 :
297 : /* if nothing failed, just recompute the parity */
298 869093 : if (failed_count == 0) {
299 586158 : raid_gen(diskmax, state->level, state->block_size, buffer);
300 586158 : return 0;
301 : }
302 :
303 : /* logs the status */
304 1006981 : for (j = 0; j < failed_count; ++j) {
305 : const char* state;
306 : const char* hash;
307 : const char* data;
308 724046 : struct snapraid_block* block = failed[j].block;
309 724046 : unsigned block_state = block_state_get(block);
310 :
311 724046 : switch (block_state) {
312 18454 : case BLOCK_STATE_DELETED : state = "delete"; break;
313 52111 : case BLOCK_STATE_CHG : state = "change"; break;
314 24012 : case BLOCK_STATE_REP : state = "replace"; break;
315 629469 : case BLOCK_STATE_BLK : state = "block"; break;
316 : /* LCOV_EXCL_START */
317 : default : state = "unknown"; break;
318 : /* LCOV_EXCL_STOP */
319 : }
320 :
321 724046 : if (hash_is_invalid(block->hash)) {
322 13841 : hash = "lost";
323 710205 : } else if (hash_is_zero(block->hash)) {
324 29636 : hash = "zero";
325 : } else {
326 680569 : hash = "known";
327 : }
328 :
329 724046 : if (failed[j].is_bad)
330 638323 : data = "bad";
331 : else
332 85723 : data = "good";
333 :
334 724046 : if (failed[j].handle) {
335 : const char* disk;
336 : const char* sub;
337 705592 : struct snapraid_handle* handle = failed[j].handle;
338 705592 : disk = handle->disk->name;
339 705592 : sub = block_file_get(block)->sub;
340 :
341 705592 : log_tag("entry:%u:%s:%s:%s:%s:%s:%u:\n", j, state, hash, data, disk, esc(sub), block_file_pos(block));
342 : } else {
343 18454 : log_tag("entry:%u:%s:%s:%s:\n", j, state, hash, data);
344 : }
345 : }
346 :
347 : /* Here we have to try two different strategies to recover, because in case the 'sync' */
348 : /* process is aborted, we don't know if the parity data is really updated just like after 'sync', */
349 : /* or if it still represents the state before the 'sync'. */
350 :
351 : /* Note that if the 'sync' ends normally, we don't have any DELETED, REP and CHG blocks */
352 : /* and the two strategies are identical */
353 :
354 : /* As first, we assume that the parity IS updated for the current state */
355 : /* and that we are going to recover the state after the last 'sync'. */
356 : /* In this case, parity contains info from BLK, REP and CHG blocks, */
357 : /* but not for DELETED. */
358 : /* We need to put in the recovering process only the bad blocks, because all the */
359 : /* others already contains the correct data read from disk, and the parity is correctly computed for them. */
360 : /* We are interested to recover BLK, REP and CHG blocks if they are marked as bad, */
361 : /* but we are not interested in DELETED ones. */
362 :
363 282935 : n = 0;
364 282935 : something_to_recover = 0; /* keep track if there is at least one block to fix */
365 1006981 : for (j = 0; j < failed_count; ++j) {
366 724046 : if (failed[j].is_bad) {
367 638323 : unsigned block_state = block_state_get(failed[j].block);
368 :
369 638323 : assert(block_state != BLOCK_STATE_DELETED); /* we cannot have bad DELETED blocks */
370 :
371 : /* if we have the hash for it */
372 638323 : if ((block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP)
373 : /* try to fetch the block using the known hash */
374 629907 : && (state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0
375 625435 : || state_search_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0)
376 : ) {
377 : /* we already have corrected it! */
378 13854 : log_tag("hash_import: Fixed entry %u\n", j);
379 : } else {
380 : /* otherwise try to recover it */
381 624469 : failed_map[n] = j;
382 624469 : ++n;
383 :
384 : /* we have something to try to recover */
385 624469 : something_to_recover = 1;
386 : }
387 : }
388 : }
389 :
390 : /* if nothing to fix */
391 282935 : if (!something_to_recover) {
392 35874 : log_tag("recover_sync:%u:%u: Skipped for already recovered\n", pos, n);
393 :
394 : /* recompute only the parity */
395 35874 : raid_gen(diskmax, state->level, state->block_size, buffer);
396 35874 : return 0;
397 : }
398 :
399 247061 : ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, buffer_zero);
400 247061 : if (ret == 0) {
401 : /* reprocess the CHG blocks, for which we don't have a hash to check */
402 : /* if they were BAD we have to use some euristics to ensure that we have recovered */
403 : /* the state after the sync. If unsure, we assume the worst case */
404 :
405 674524 : for (j = 0; j < failed_count; ++j) {
406 : /* we take care only of BAD blocks we have to write back */
407 479693 : if (failed[j].is_bad) {
408 466179 : unsigned block_state = block_state_get(failed[j].block);
409 :
410 : /* BLK and REP blocks are always OK, because at this point */
411 : /* we have already checked their hash */
412 466179 : if (block_state != BLOCK_STATE_CHG) {
413 457763 : assert(block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP);
414 457763 : continue;
415 : }
416 :
417 : /* for CHG blocks we have to 'guess' if they are correct or not */
418 :
419 : /* if the hash is invalid we cannot check the result */
420 : /* this could happen if we have lost this information */
421 : /* after an aborted sync */
422 8416 : if (hash_is_invalid(failed[j].block->hash)) {
423 : /* it may contain garbage */
424 0 : failed[j].is_outofdate = 1;
425 :
426 0 : log_tag("hash_unknown: Unknown hash on entry %u\n", j);
427 8416 : } else if (hash_is_zero(failed[j].block->hash)) {
428 : /* if the block is not filled with 0, we are sure to have */
429 : /* restored it to the state after the 'sync' */
430 : /* instead, if the block is filled with 0, it could be either that the */
431 : /* block after the sync is really filled by 0, or that */
432 : /* we restored the block before the 'sync'. */
433 8416 : if (memcmp(buffer[failed[j].index], buffer_zero, state->block_size) == 0) {
434 : /* it may contain garbage */
435 4205 : failed[j].is_outofdate = 1;
436 :
437 4205 : log_tag("hash_unknown: Maybe old zero on entry %u\n", j);
438 : }
439 : } else {
440 : /* if the hash is different than the previous one, we are sure to have */
441 : /* restored it to the state after the 'sync' */
442 : /* instead, if the hash matches, it could be either that the */
443 : /* block after the sync has this hash, or that */
444 : /* we restored the block before the 'sync'. */
445 0 : if (blockcmp(state, rehash, failed[j].block, buffer[failed[j].index], buffer_zero) == 0) {
446 : /* it may contain garbage */
447 0 : failed[j].is_outofdate = 1;
448 :
449 0 : log_tag("hash_unknown: Maybe old data on entry %u\n", j);
450 : }
451 : }
452 : }
453 : }
454 :
455 194831 : return 0;
456 : }
457 52230 : if (ret > 0)
458 9163 : error += ret;
459 :
460 52230 : if (ret < 0)
461 43067 : log_tag("recover_sync:%u:%u: Failed with no attempts\n", pos, n);
462 : else
463 9163 : log_tag("recover_sync:%u:%u: Failed with %d attempts\n", pos, n, ret);
464 :
465 : /* Now assume that the parity IS NOT updated at the current state, */
466 : /* but still represent the state before the last 'sync' process. */
467 : /* In this case, parity contains info from BLK, REP (old version), CHG (old version) and DELETED blocks, */
468 : /* but not for REP (new version) and CHG (new version). */
469 : /* We are interested to recover BLK ones marked as bad, */
470 : /* but we are not interested to recover CHG (new version) and REP (new version) blocks, */
471 : /* even if marked as bad, because we don't have parity for them and it's just impossible, */
472 : /* and we are not interested to recover DELETED ones. */
473 52230 : n = 0;
474 52230 : something_to_recover = 0; /* keep track if there is at least one block to fix */
475 52230 : something_unsynced = 0; /* keep track if we have some unsynced info to process */
476 232353 : for (j = 0; j < failed_count; ++j) {
477 180123 : unsigned block_state = block_state_get(failed[j].block);
478 :
479 180123 : if (block_state == BLOCK_STATE_DELETED
480 171079 : || block_state == BLOCK_STATE_CHG
481 171079 : || block_state == BLOCK_STATE_REP
482 : ) {
483 : /* If the block is CHG, REP or DELETED, we don't have the original content of block, */
484 : /* and we must try to recover it. */
485 : /* This apply to CHG and REP blocks even if they are not marked bad, */
486 : /* because the parity is computed with old content, and not with the new one. */
487 : /* Note that this recovering is done just to make possible to recover any other BLK one, */
488 : /* we are not really interested in DELETED, CHG (old version) and REP (old version). */
489 9044 : something_unsynced = 1;
490 :
491 9044 : if (block_state == BLOCK_STATE_CHG
492 0 : && hash_is_zero(failed[j].block->hash)
493 : ) {
494 : /* If the block was a ZERO block, restore it to the original 0 as before the 'sync' */
495 : /* We do this to just allow recovering of other BLK ones */
496 :
497 0 : memset(buffer[failed[j].index], 0, state->block_size);
498 : /* note that from now the buffer is definitively lost */
499 : /* we can do this only because it's the last retry of recovering */
500 :
501 : /* try to fetch the old block using the old hash for CHG and DELETED blocks */
502 9044 : } else if ((block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_DELETED)
503 9044 : && hash_is_real(failed[j].block->hash)
504 9044 : && state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0) {
505 :
506 : /* note that from now the buffer is definitively lost */
507 : /* we can do this only because it's the last retry of recovering */
508 : } else {
509 : /* otherwise try to recover it */
510 9044 : failed_map[n] = j;
511 9044 : ++n;
512 :
513 : /* note that we don't set something_to_recover, because we are */
514 : /* not really interested to recover *only* old blocks. */
515 : }
516 :
517 : /* avoid to use the hash of this block to verify the recovering */
518 : /* this applies to REP blocks because we are going to recover the old state */
519 : /* and the REP hash represent the new one */
520 : /* it also applies to CHG and DELETE blocks because we want to have */
521 : /* a succesful recovering only if a BLK one is matching */
522 9044 : failed[j].is_outofdate = 1;
523 171079 : } else if (failed[j].is_bad) {
524 : /* If the block is bad we don't know its content, and we try to recover it */
525 : /* At this point, we can have only BLK ones */
526 :
527 171079 : assert(block_state == BLOCK_STATE_BLK);
528 :
529 : /* we have something we are interested to recover */
530 171079 : something_to_recover = 1;
531 :
532 : /* we try to recover it */
533 171079 : failed_map[n] = j;
534 171079 : ++n;
535 : }
536 : }
537 :
538 : /* if nothing to fix, we just don't try */
539 : /* if nothing unsynced we also don't retry, because it's the same try as before */
540 52230 : if (something_to_recover && something_unsynced) {
541 9044 : ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, buffer_zero);
542 9044 : if (ret == 0) {
543 : /* reprocess the REP and CHG blocks, for which we have recovered and old state */
544 : /* that we don't want to save into disk */
545 : /* we have already marked them, but we redo it for logging */
546 :
547 13566 : for (j = 0; j < failed_count; ++j) {
548 : /* we take care only of BAD blocks we have to write back */
549 9044 : if (failed[j].is_bad) {
550 4522 : unsigned block_state = block_state_get(failed[j].block);
551 :
552 4522 : if (block_state == BLOCK_STATE_CHG
553 4522 : || block_state == BLOCK_STATE_REP
554 : ) {
555 : /* mark that we have restored an old state */
556 : /* and we don't want to write it to the disk */
557 0 : failed[j].is_outofdate = 1;
558 :
559 0 : log_tag("hash_unknown: Surely old data on entry %u\n", j);
560 : }
561 : }
562 : }
563 :
564 4522 : return 0;
565 : }
566 4522 : if (ret > 0)
567 0 : error += ret;
568 :
569 9044 : if (ret < 0)
570 4522 : log_tag("recover_unsync:%u:%u: Failed with no attempts\n", pos, n);
571 : else
572 0 : log_tag("recover_unsync:%u:%u: Failed with %d attempts\n", pos, n, ret);
573 : } else {
574 43186 : log_tag("recover_unsync:%u:%u: Skipped for%s%s\n", pos, n,
575 : !something_to_recover ? " nothing to recover" : "",
576 : !something_unsynced ? " nothing unsynched" : ""
577 : );
578 : }
579 :
580 : /* return the number of failed attempts, or -1 if no strategy */
581 47708 : if (error)
582 4641 : return error;
583 : else
584 43067 : return -1;
585 : }
586 :
587 : /**
588 : * Post process all the files at the specified block index ::i.
589 : * For each file, if we are at the last block, closes it,
590 : * adjust the timestamp, and prints the result.
591 : */
592 933330 : static int file_post(struct snapraid_state* state, int fix, unsigned i, struct snapraid_handle* handle, unsigned diskmax)
593 : {
594 : unsigned j;
595 : int ret;
596 :
597 : /* for all the files prints the final status, and does the final time fix */
598 : /* we also ensure to close files after processing the last block */
599 6533310 : for (j = 0; j < diskmax; ++j) {
600 : struct snapraid_block* block;
601 : struct snapraid_disk* disk;
602 : struct snapraid_file* collide_file;
603 : struct snapraid_file* file;
604 : char path[PATH_MAX];
605 : uint64_t inode;
606 :
607 5599980 : disk = handle[j].disk;
608 5599980 : if (!disk) {
609 : /* if no disk, nothing to do */
610 3430290 : continue;
611 : }
612 :
613 5585967 : block = disk_block_get(disk, i);
614 5585967 : if (!block_has_file(block)) {
615 : /* if no file, nothing to do */
616 210457 : continue;
617 : }
618 :
619 5375510 : file = block_file_get(block);
620 5375510 : pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
621 :
622 : /* if it isn't the last block in the file */
623 5375510 : if (!block_is_last(block)) {
624 : /* nothing to do */
625 3191807 : continue;
626 : }
627 :
628 : /* if the file is excluded, we have nothing to adjust as the file is never written */
629 2183703 : if (file_flag_has(file, FILE_IS_EXCLUDED)
630 2086038 : || (state->opt.syncedonly && file_flag_has(file, FILE_IS_UNSYNCED))) {
631 : /* nothing to do, but close the file */
632 : goto close_and_continue;
633 : }
634 :
635 : /* if the file is opened and not excluded, it must be the correct one */
636 : /* or at least pointing to NULL, in case it cannot be opened. */
637 2086038 : if (handle[j].file != 0 && handle[j].file != file) {
638 : /* LCOV_EXCL_START */
639 : log_fatal("Internal inconsistency in opened file for block %u\n", block->parity_pos);
640 : exit(EXIT_FAILURE);
641 : /* LCOV_EXCL_STOP */
642 : }
643 :
644 : /* finish the fix process if it's the last block of the files */
645 2086038 : if (fix) {
646 : /* mark that we finished with this file */
647 : /* to identify later any NOT finished ones */
648 767408 : file_flag_set(file, FILE_IS_FINISHED);
649 :
650 : /* if the file is damaged, meaning that a fix failed */
651 767408 : if (file_flag_has(file, FILE_IS_DAMAGED)) {
652 : /* rename it to .unrecoverable */
653 : char path_to[PATH_MAX];
654 :
655 64977 : pathprint(path_to, sizeof(path_to), "%s%s.unrecoverable", disk->dir, file->sub);
656 :
657 : /* ensure to close the file before renaming */
658 64977 : ret = handle_close(&handle[j]);
659 64977 : if (ret != 0) {
660 : /* LCOV_EXCL_START */
661 : log_fatal("Error closing '%s'. %s.\n", path, strerror(errno));
662 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
663 : return -1;
664 : /* LCOV_EXCL_STOP */
665 : }
666 :
667 64977 : ret = rename(path, path_to);
668 64977 : if (ret != 0) {
669 : /* LCOV_EXCL_START */
670 : log_fatal("Error renaming '%s' to '%s'. %s.\n", path, path_to, strerror(errno));
671 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
672 : return -1;
673 : /* LCOV_EXCL_STOP */
674 : }
675 :
676 64977 : log_tag("status:unrecoverable:%s:%s\n", disk->name, esc(file->sub));
677 64977 : msg_info("unrecoverable %s\n", path);
678 :
679 : /* and do not set the time if damaged */
680 64977 : goto close_and_continue;
681 : }
682 :
683 : /* if the file is not fixed, meaning that it is untouched */
684 702431 : if (!file_flag_has(file, FILE_IS_FIXED)) {
685 : /* nothing to do, but close the file */
686 582821 : goto close_and_continue;
687 : }
688 :
689 119610 : log_tag("status:recovered:%s:%s\n", disk->name, esc(file->sub));
690 119610 : msg_info("recovered %s\n", path);
691 :
692 119610 : inode = handle[j].st.st_ino;
693 :
694 : /* search for the corresponding inode */
695 119610 : collide_file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
696 :
697 : /* if the inode is already in the database and it refers at a different file name, */
698 : /* we can fix the file time ONLY if the time and size allow to differentiate */
699 : /* between the two files */
700 :
701 : /* for example, suppose we delete a bunch of files with all the same size and time, */
702 : /* when recreating them the inodes may be reused in a different order, */
703 : /* and at the next sync some files may have matching inode/size/time even if different name */
704 : /* not allowing sync to detect that the file is changed and not renamed */
705 119610 : if (!collide_file /* if not in the database, there is no collision */
706 37284 : || strcmp(collide_file->sub, file->sub) == 0 /* if the name is the same, it's the right collision */
707 22374 : || collide_file->size != file->size /* if the size is different, the collision is identified */
708 1 : || collide_file->mtime_sec != file->mtime_sec /* if the mtime is different, the collision is identified */
709 1 : || collide_file->mtime_nsec != file->mtime_nsec /* same for mtime_nsec */
710 : ) {
711 : /* set the original modification time */
712 119610 : ret = handle_utime(&handle[j]);
713 239220 : if (ret == -1) {
714 : /* LCOV_EXCL_START */
715 : /* mark the file as damaged */
716 : file_flag_set(file, FILE_IS_DAMAGED);
717 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
718 : return -1;
719 : /* LCOV_EXCL_STOP */
720 : }
721 : } else {
722 0 : log_tag("collision:%s:%s:%s: Not setting modification time to avoid inode collision\n", disk->name, esc(file->sub), esc(collide_file->sub));
723 : }
724 : } else {
725 : /* we are not fixing, but only checking */
726 : /* print just the final status */
727 1318630 : if (file_flag_has(file, FILE_IS_DAMAGED)) {
728 9328 : if (state->opt.auditonly) {
729 2759 : log_tag("status:damaged:%s:%s\n", disk->name, esc(file->sub));
730 2759 : msg_info("damaged %s\n", path);
731 : } else {
732 6569 : log_tag("status:unrecoverable:%s:%s\n", disk->name, esc(file->sub));
733 6569 : msg_info("unrecoverable %s\n", path);
734 : }
735 1309302 : } else if (file_flag_has(file, FILE_IS_FIXED)) {
736 76455 : log_tag("status:recoverable:%s:%s\n", disk->name, esc(file->sub));
737 76455 : msg_info("recoverable %s\n", path);
738 : } else {
739 1232847 : log_tag("status:correct:%s:%s\n", disk->name, esc(file->sub));
740 : /* we don't use msg_verbose() because it also goes into the log */
741 1232847 : if (msg_level >= MSG_VERBOSE)
742 23188 : msg_info("correct %s\n", path);
743 : }
744 : }
745 :
746 : close_and_continue:
747 : /* if the opened file is the correct one, close it */
748 : /* in case of excluded and fragmented files it's possible */
749 : /* that the opened file is not the current one */
750 2183703 : if (handle[j].file == file) {
751 : /* ensure to close the file just after finishing with it */
752 1986275 : ret = handle_close(&handle[j]);
753 1986275 : if (ret != 0) {
754 : /* LCOV_EXCL_START */
755 : log_fatal("Error closing '%s'. %s.\n", path, strerror(errno));
756 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
757 : return -1;
758 : /* LCOV_EXCL_STOP */
759 : }
760 : }
761 : }
762 :
763 933330 : return 0;
764 : }
765 :
766 : /**
767 : * Check if we have to process the specified block index ::i.
768 : */
769 1866660 : static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct snapraid_handle* handle, unsigned diskmax)
770 : {
771 : int one_tocheck;
772 : snapraid_info info;
773 : unsigned j;
774 :
775 : /* get block specific info */
776 1866660 : info = info_get(&state->infoarr, i);
777 :
778 : /* if we filter for only bad blocks */
779 1866660 : if (state->opt.badonly) {
780 : /* skip if this is not bad */
781 18684 : if (!info_get_bad(info))
782 15078 : return 0;
783 : }
784 :
785 : /* for each disk */
786 1851582 : one_tocheck = 0;
787 2299380 : for (j = 0; j < diskmax; ++j) {
788 : struct snapraid_block* block;
789 :
790 : /* if no disk, nothing to check */
791 2264338 : if (!handle[j].disk)
792 708 : continue;
793 :
794 2263630 : block = disk_block_get(handle[j].disk, i);
795 :
796 : /* try to recover all files, even the ones without hash */
797 : /* because in some cases we can recover also them */
798 2263630 : if (block_has_file(block)
799 2111788 : && !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */
800 : ) {
801 1816540 : one_tocheck = 1;
802 1816540 : break;
803 : }
804 : }
805 :
806 : /* if no block to check skip */
807 1851582 : if (!one_tocheck)
808 35042 : return 0;
809 :
810 1816540 : return 1;
811 : }
812 :
813 118 : static int state_check_process(struct snapraid_state* state, int fix, struct snapraid_parity_handle** parity, block_off_t blockstart, block_off_t blockmax)
814 : {
815 : struct snapraid_handle* handle;
816 : unsigned diskmax;
817 : block_off_t i;
818 : unsigned j;
819 : void* buffer_alloc;
820 : void** buffer;
821 : unsigned buffermax;
822 : int ret;
823 : data_off_t countsize;
824 : block_off_t countpos;
825 : block_off_t countmax;
826 : unsigned error;
827 : unsigned unrecoverable_error;
828 : unsigned recovered_error;
829 : struct failed_struct* failed;
830 : unsigned* failed_map;
831 : unsigned l;
832 :
833 118 : handle = handle_map(state, &diskmax);
834 :
835 : /* we need 1 * data + 2 * parity + 1 * zero */
836 118 : buffermax = diskmax + 2 * state->level + 1;
837 :
838 118 : buffer = malloc_nofail_vector_align(diskmax, buffermax, state->block_size, &buffer_alloc);
839 118 : if (!state->opt.skip_self)
840 0 : mtest_vector(buffermax, state->block_size, buffer);
841 :
842 : /* fill up the zero buffer */
843 118 : memset(buffer[buffermax - 1], 0, state->block_size);
844 118 : raid_zero(buffer[buffermax - 1]);
845 :
846 118 : failed = malloc_nofail(diskmax * sizeof(struct failed_struct));
847 118 : failed_map = malloc_nofail(diskmax * sizeof(unsigned));
848 :
849 118 : error = 0;
850 118 : unrecoverable_error = 0;
851 118 : recovered_error = 0;
852 :
853 : /* first count the number of blocks to process */
854 118 : countmax = 0;
855 933448 : for (i = blockstart; i < blockmax; ++i) {
856 933330 : if (!block_is_enabled(state, i, handle, diskmax))
857 25060 : continue;
858 908270 : ++countmax;
859 : }
860 :
861 : /* check all the blocks in files */
862 118 : countsize = 0;
863 118 : countpos = 0;
864 118 : state_progress_begin(state, blockstart, blockmax, countmax);
865 933448 : for (i = blockstart; i < blockmax; ++i) {
866 : unsigned failed_count;
867 : int valid_parity;
868 : snapraid_info info;
869 : int rehash;
870 :
871 933330 : if (!block_is_enabled(state, i, handle, diskmax)) {
872 : /* post process the files */
873 25060 : ret = file_post(state, fix, i, handle, diskmax);
874 25060 : if (ret == -1) {
875 0 : log_fatal("Stopping at block %u\n", i);
876 0 : ++unrecoverable_error;
877 0 : goto bail;
878 : }
879 :
880 : /* and now continue with the next block */
881 25060 : continue;
882 : }
883 :
884 : /* If we have valid parity, and it makes sense to check its content. */
885 : /* If we already know that the parity is invalid, we just read the file */
886 : /* but we don't report parity errors */
887 : /* Note that with auditonly, we anyway skip the full parity check, */
888 : /* because we also don't read it at all */
889 908270 : valid_parity = 1;
890 :
891 : /* keep track of the number of failed blocks */
892 908270 : failed_count = 0;
893 :
894 : /* get block specific info */
895 908270 : info = info_get(&state->infoarr, i);
896 :
897 : /* if we have to use the old hash */
898 908270 : rehash = info_get_rehash(info);
899 :
900 : /* for each disk, process the block */
901 6357890 : for (j = 0; j < diskmax; ++j) {
902 : int read_size;
903 : unsigned char hash[HASH_SIZE];
904 : struct snapraid_disk* disk;
905 : struct snapraid_block* block;
906 : struct snapraid_file* file;
907 : unsigned block_state;
908 :
909 : /* if the disk position is not used */
910 5449620 : disk = handle[j].disk;
911 5449620 : if (!disk) {
912 : /* use an empty block */
913 14013 : memset(buffer[j], 0, state->block_size);
914 946594 : continue;
915 : }
916 :
917 : /* if the disk block is not used */
918 5435607 : block = disk_block_get(disk, i);
919 5435607 : if (block == BLOCK_EMPTY) {
920 : /* use an empty block */
921 188750 : memset(buffer[j], 0, state->block_size);
922 188750 : continue;
923 : }
924 :
925 : /* get the state of the block */
926 5246857 : block_state = block_state_get(block);
927 :
928 : /* if the parity is not valid */
929 5246857 : if (block_has_invalid_parity(block)) {
930 : /* mark the parity as invalid, and don't try to check/fix it */
931 : /* because it will be recomputed at the next sync */
932 94577 : valid_parity = 0;
933 : /* follow */
934 : }
935 :
936 : /* if the block is DELETED */
937 5246857 : if (block_state == BLOCK_STATE_DELETED) {
938 : /* use an empty block */
939 18454 : memset(buffer[j], 0, state->block_size);
940 :
941 : /* store it in the failed set, because potentially */
942 : /* the parity may be still computed with the previous content */
943 18454 : failed[failed_count].is_bad = 0;
944 18454 : failed[failed_count].is_outofdate = 0;
945 18454 : failed[failed_count].index = j;
946 18454 : failed[failed_count].block = block;
947 18454 : failed[failed_count].handle = 0;
948 18454 : ++failed_count;
949 18454 : continue;
950 : }
951 :
952 : /* get the file of this block */
953 5228403 : file = block_file_get(block);
954 :
955 : /* if we are only hashing, we can skip excluded files and don't even read them */
956 5228403 : if (state->opt.auditonly && file_flag_has(file, FILE_IS_EXCLUDED)) {
957 : /* use an empty block */
958 : /* in true, this is unnecessary, becase we are not checking any parity */
959 : /* but we keep it for completeness */
960 0 : memset(buffer[j], 0, state->block_size);
961 0 : continue;
962 : }
963 :
964 : /* if the file is closed or different than the current one */
965 5228403 : if (handle[j].file == 0 || handle[j].file != file) {
966 : /* close the old one, if any */
967 2255952 : ret = handle_close(&handle[j]);
968 2255952 : if (ret == -1) {
969 : /* LCOV_EXCL_START */
970 : log_fatal("DANGER! Unexpected close error in a data disk, it isn't possible to check.\n");
971 : log_fatal("Stopping at block %u\n", i);
972 : ++unrecoverable_error;
973 : goto bail;
974 : /* LCOV_EXCL_STOP */
975 : }
976 :
977 : /* if fixing, and the file is not excluded, we must open for writing */
978 2255952 : if (fix && !file_flag_has(file, FILE_IS_EXCLUDED)) {
979 : /* if fixing, create the file, open for writing and resize if required */
980 773174 : ret = handle_create(&handle[j], file, state->file_mode);
981 773174 : if (ret == -1) {
982 : /* LCOV_EXCL_START */
983 : if (errno == EACCES) {
984 : log_fatal("WARNING! Please give write permission to the file.\n");
985 : } else {
986 : log_fatal("DANGER! Without a working data disk, it isn't possible to fix errors on it.\n");
987 : }
988 : log_fatal("Stopping at block %u\n", i);
989 : ++unrecoverable_error;
990 : goto bail;
991 : /* LCOV_EXCL_STOP */
992 : }
993 :
994 : /* check if the file was just created */
995 1546348 : if (handle[j].created != 0) {
996 : /* if fragmented, it may be reopened, so remember that the file */
997 : /* was originally missing */
998 112974 : file_flag_set(file, FILE_IS_CREATED);
999 : }
1000 : } else {
1001 : /* otherwise, open the file only for reading */
1002 1482778 : ret = handle_open(&handle[j], file, state->file_mode,
1003 1482778 : log_error, state->opt.expected_missing ? log_expected : 0);
1004 1482778 : if (ret == -1) {
1005 : /* save the failed block for the check/fix */
1006 189292 : failed[failed_count].is_bad = 1;
1007 189292 : failed[failed_count].is_outofdate = 0;
1008 189292 : failed[failed_count].index = j;
1009 189292 : failed[failed_count].block = block;
1010 189292 : failed[failed_count].handle = &handle[j];
1011 189292 : ++failed_count;
1012 :
1013 189292 : log_tag("error:%u:%s:%s: Open error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block));
1014 189292 : ++error;
1015 189292 : continue;
1016 : }
1017 : }
1018 :
1019 : /* if it's the first open, and not excluded */
1020 2066660 : if (!file_flag_has(file, FILE_IS_OPENED)
1021 2051280 : && !file_flag_has(file, FILE_IS_EXCLUDED)) {
1022 :
1023 : /* check if the file is changed */
1024 2011179 : if (handle[j].st.st_size != block_file_get(block)->size
1025 1818097 : || handle[j].st.st_mtime != block_file_get(block)->mtime_sec
1026 1795473 : || STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
1027 : /* don't check the inode to support filesystem without persistent inodes */
1028 : ) {
1029 : /* report that the file is not synced */
1030 215706 : file_flag_set(file, FILE_IS_UNSYNCED);
1031 : }
1032 : }
1033 :
1034 : /* if it's the first open, and not excluded and larger */
1035 2066660 : if (!file_flag_has(file, FILE_IS_OPENED)
1036 2051280 : && !file_flag_has(file, FILE_IS_EXCLUDED)
1037 2011179 : && !(state->opt.syncedonly && file_flag_has(file, FILE_IS_UNSYNCED))
1038 2011179 : && handle[j].st.st_size > file->size
1039 : ) {
1040 8665 : log_error("File '%s' is larger than expected.\n", handle[j].path);
1041 8665 : log_tag("error:%u:%s:%s: Size error\n", i, disk->name, esc(file->sub));
1042 8665 : ++error;
1043 :
1044 8665 : if (fix) {
1045 3195 : ret = handle_truncate(&handle[j], file);
1046 3195 : if (ret == -1) {
1047 : /* LCOV_EXCL_START */
1048 : log_fatal("DANGER! Unexpected truncate error in a data disk, it isn't possible to fix.\n");
1049 : log_fatal("Stopping at block %u\n", i);
1050 : ++unrecoverable_error;
1051 : goto bail;
1052 : /* LCOV_EXCL_STOP */
1053 : }
1054 :
1055 3195 : log_tag("fixed:%u:%s:%s: Fixed size\n", i, disk->name, esc(file->sub));
1056 3195 : ++recovered_error;
1057 : }
1058 : }
1059 :
1060 : /* mark the file as opened at least one time */
1061 : /* this is used to avoid to check the unsynced and size */
1062 : /* more than one time, in case the file is reopened later */
1063 2066660 : file_flag_set(file, FILE_IS_OPENED);
1064 : }
1065 :
1066 : /* read from the file */
1067 5039111 : read_size = handle_read(&handle[j], block, buffer[j], state->block_size,
1068 5039111 : log_error, state->opt.expected_missing ? log_expected : 0);
1069 5039111 : if (read_size == -1) {
1070 : /* save the failed block for the check/fix */
1071 442133 : failed[failed_count].is_bad = 1; /* it's bad because we cannot read it */
1072 442133 : failed[failed_count].is_outofdate = 0;
1073 442133 : failed[failed_count].index = j;
1074 442133 : failed[failed_count].block = block;
1075 442133 : failed[failed_count].handle = &handle[j];
1076 442133 : ++failed_count;
1077 :
1078 442133 : log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block));
1079 442133 : ++error;
1080 442133 : continue;
1081 : }
1082 :
1083 4596978 : countsize += read_size;
1084 :
1085 : /* always insert CHG blocks, the repair functions needs all of them */
1086 : /* because the parity may be still referring at the old state */
1087 : /* and the repair must be aware of it */
1088 4596978 : if (block_state == BLOCK_STATE_CHG) {
1089 : /* we DO NOT mark them as bad to avoid to overwrite them with wrong data. */
1090 : /* if we don't have a hash, we always assume the first read of the block correct. */
1091 43695 : failed[failed_count].is_bad = 0; /* we assume the CHG block correct */
1092 43695 : failed[failed_count].is_outofdate = 0;
1093 43695 : failed[failed_count].index = j;
1094 43695 : failed[failed_count].block = block;
1095 43695 : failed[failed_count].handle = &handle[j];
1096 43695 : ++failed_count;
1097 43695 : continue;
1098 : }
1099 :
1100 4553283 : assert(block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP);
1101 :
1102 : /* compute the hash of the block just read */
1103 4553283 : if (rehash) {
1104 286339 : memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
1105 : } else {
1106 4266944 : memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
1107 : }
1108 :
1109 : /* compare the hash */
1110 4553283 : if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
1111 : /* save the failed block for the check/fix */
1112 12670 : failed[failed_count].is_bad = 1; /* it's bad because the hash doesn't match */
1113 12670 : failed[failed_count].is_outofdate = 0;
1114 12670 : failed[failed_count].index = j;
1115 12670 : failed[failed_count].block = block;
1116 12670 : failed[failed_count].handle = &handle[j];
1117 12670 : ++failed_count;
1118 :
1119 12670 : log_tag("error:%u:%s:%s: Data error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block));
1120 12670 : ++error;
1121 12670 : continue;
1122 : }
1123 :
1124 : /* always insert REP blocks, the repair functions needs all of them */
1125 : /* because the parity may be still referring at the old state */
1126 : /* and the repair must be aware of it */
1127 4540613 : if (block_state == BLOCK_STATE_REP) {
1128 23574 : failed[failed_count].is_bad = 0; /* it's not bad */
1129 23574 : failed[failed_count].is_outofdate = 0;
1130 23574 : failed[failed_count].index = j;
1131 23574 : failed[failed_count].block = block;
1132 23574 : failed[failed_count].handle = &handle[j];
1133 23574 : ++failed_count;
1134 23574 : continue;
1135 : }
1136 : }
1137 :
1138 : /* now read and check the parity if requested */
1139 908270 : if (!state->opt.auditonly) {
1140 : void* buffer_recov[LEV_MAX];
1141 : void* buffer_zero;
1142 :
1143 : /* buffers for parity read and not computed */
1144 4138380 : for (l = 0; l < state->level; ++l)
1145 3269287 : buffer_recov[l] = buffer[diskmax + state->level + l];
1146 2814364 : for (; l < LEV_MAX; ++l)
1147 1945271 : buffer_recov[l] = 0;
1148 :
1149 : /* the zero buffer is the last one */
1150 869093 : buffer_zero = buffer[buffermax - 1];
1151 :
1152 : /* read the parity */
1153 4138380 : for (l = 0; l < state->level; ++l) {
1154 3269287 : if (parity[l]) {
1155 3260088 : ret = parity_read(parity[l], i, buffer_recov[l], state->block_size, log_error);
1156 3260088 : if (ret == -1) {
1157 90270 : buffer_recov[l] = 0; /* no parity to use */
1158 :
1159 90270 : log_tag("parity_error:%u:%s: Read error\n", i, lev_config_name(l));
1160 90270 : ++error;
1161 : }
1162 : } else {
1163 9199 : buffer_recov[l] = 0;
1164 : }
1165 : }
1166 :
1167 : /* try all the recovering strategies */
1168 869093 : ret = repair(state, rehash, i, diskmax, failed, failed_map, failed_count, buffer, buffer_recov, buffer_zero);
1169 869093 : if (ret != 0) {
1170 : /* increment the number of errors */
1171 47708 : if (ret > 0)
1172 4641 : error += ret;
1173 47708 : ++unrecoverable_error;
1174 :
1175 : /* print a list of all the errors in files */
1176 218787 : for (j = 0; j < failed_count; ++j) {
1177 171079 : if (failed[j].is_bad)
1178 166557 : log_tag("unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].handle->disk->name, esc(block_file_get(failed[j].block)->sub), block_file_pos(failed[j].block));
1179 : }
1180 :
1181 : /* keep track of damaged files */
1182 218787 : for (j = 0; j < failed_count; ++j) {
1183 171079 : if (failed[j].is_bad)
1184 166557 : file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
1185 : }
1186 : } else {
1187 : /* now counts partial recovers */
1188 : /* note that this could happen only when we have an incomplete 'sync' */
1189 : /* and that we have recovered is the state before the 'sync' */
1190 821385 : int partial_recover_error = 0;
1191 :
1192 : /* print a list of all the errors in files */
1193 1374352 : for (j = 0; j < failed_count; ++j) {
1194 552967 : if (failed[j].is_bad && failed[j].is_outofdate) {
1195 4205 : ++partial_recover_error;
1196 4205 : log_tag("unrecoverable:%u:%s:%s: Unrecoverable unsynced error at position %u\n", i, failed[j].handle->disk->name, esc(block_file_get(failed[j].block)->sub), block_file_pos(failed[j].block));
1197 : }
1198 : }
1199 821385 : if (partial_recover_error != 0) {
1200 4205 : error += partial_recover_error;
1201 4205 : ++unrecoverable_error;
1202 : }
1203 :
1204 : /* now check parities, but only if all the blocks have it computed */
1205 : /* if you check/fix after a partial sync, it's OK to have parity errors on the blocks with invalid parity */
1206 : /* and doesn't make sense to try to fix it */
1207 821385 : if (valid_parity) {
1208 : /* check the parity */
1209 3695991 : for (l = 0; l < state->level; ++l) {
1210 2936305 : if (buffer_recov[l] != 0 && memcmp(buffer_recov[l], buffer[diskmax + l], state->block_size) != 0) {
1211 10631 : buffer_recov[l] = 0;
1212 :
1213 10631 : log_tag("parity_error:%u:%s: Data error\n", i, lev_config_name(l));
1214 10631 : ++error;
1215 : }
1216 : }
1217 : }
1218 :
1219 : /* now writes recovered files */
1220 821385 : if (fix) {
1221 : /* update the fixed files */
1222 642366 : for (j = 0; j < failed_count; ++j) {
1223 : /* nothing to do if it doesn't need recovering */
1224 332980 : if (!failed[j].is_bad)
1225 40318 : continue;
1226 :
1227 : /* do not fix if the file is excluded */
1228 292662 : if (file_flag_has(block_file_get(failed[j].block), FILE_IS_EXCLUDED)
1229 288068 : || (state->opt.syncedonly && file_flag_has(block_file_get(failed[j].block), FILE_IS_UNSYNCED)))
1230 4594 : continue;
1231 :
1232 288068 : ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size);
1233 288068 : if (ret == -1) {
1234 : /* LCOV_EXCL_START */
1235 : /* mark the file as damaged */
1236 : file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
1237 :
1238 : if (errno == EACCES) {
1239 : log_fatal("WARNING! Please give write permission to the file.\n");
1240 : } else {
1241 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1242 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1243 : }
1244 : log_fatal("Stopping at block %u\n", i);
1245 : ++unrecoverable_error;
1246 : goto bail;
1247 : /* LCOV_EXCL_STOP */
1248 : }
1249 :
1250 : /* if we are not sure that the recovered content is uptodate */
1251 288068 : if (failed[j].is_outofdate) {
1252 : /* mark the file as damaged */
1253 4205 : file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
1254 4205 : continue;
1255 : }
1256 :
1257 : /* mark the file as containing some fixes */
1258 : /* note that it could be also marked as damaged in other iterations */
1259 283863 : file_flag_set(block_file_get(failed[j].block), FILE_IS_FIXED);
1260 :
1261 283863 : log_tag("fixed:%u:%s:%s: Fixed data error at position %u\n", i, failed[j].handle->disk->name, esc(block_file_get(failed[j].block)->sub), block_file_pos(failed[j].block));
1262 283863 : ++recovered_error;
1263 : }
1264 :
1265 : /* update parity only if all the blocks have it computed */
1266 : /* if you check/fix after a partial sync, you do not want to fix parity */
1267 : /* for blocks that are going to have it computed in the sync completion */
1268 309386 : if (valid_parity) {
1269 : /* update the parity */
1270 929172 : for (l = 0; l < state->level; ++l) {
1271 668958 : if (buffer_recov[l] == 0 && parity[l] != 0) {
1272 62528 : ret = parity_write(parity[l], i, buffer[diskmax + l], state->block_size);
1273 62528 : if (ret == -1) {
1274 : /* LCOV_EXCL_START */
1275 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1276 : log_fatal("WARNING! Without a working %s disk, it isn't possible to fix errors on it.\n", lev_name(l));
1277 : log_fatal("Stopping at block %u\n", i);
1278 : ++unrecoverable_error;
1279 : goto bail;
1280 : /* LCOV_EXCL_STOP */
1281 : }
1282 :
1283 62528 : log_tag("parity_fixed:%u:%s: Fixed data error\n", i, lev_config_name(l));
1284 62528 : ++recovered_error;
1285 : }
1286 : }
1287 : }
1288 : } else {
1289 : /* if we are not fixing, we just set the FIXED flag */
1290 : /* meaning that we could fix this file if we try */
1291 731986 : for (j = 0; j < failed_count; ++j) {
1292 219987 : if (failed[j].is_bad) {
1293 179104 : file_flag_set(block_file_get(failed[j].block), FILE_IS_FIXED);
1294 : }
1295 : }
1296 : }
1297 : }
1298 : } else {
1299 : /* if we are not checking, we just set the DAMAGED flag */
1300 : /* to report that the file is damaged, and we don't know if we can fix it */
1301 44949 : for (j = 0; j < failed_count; ++j) {
1302 5772 : if (failed[j].is_bad) {
1303 5772 : file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
1304 : }
1305 : }
1306 : }
1307 :
1308 : /* post process the files */
1309 908270 : ret = file_post(state, fix, i, handle, diskmax);
1310 908270 : if (ret == -1) {
1311 0 : log_fatal("Stopping at block %u\n", i);
1312 0 : ++unrecoverable_error;
1313 0 : goto bail;
1314 : }
1315 :
1316 : /* count the number of processed block */
1317 908270 : ++countpos;
1318 :
1319 : /* progress */
1320 908270 : if (state_progress(state, i, countpos, countmax, countsize)) {
1321 : /* LCOV_EXCL_START */
1322 : break;
1323 : /* LCOV_EXCL_STOP */
1324 : }
1325 : }
1326 :
1327 : /* for each disk, recover empty files, symlinks and empty dirs */
1328 826 : for (i = 0; i < diskmax; ++i) {
1329 : tommy_node* node;
1330 : struct snapraid_disk* disk;
1331 :
1332 708 : if (!handle[i].disk)
1333 3 : continue;
1334 :
1335 : /* for each empty file in the disk */
1336 705 : disk = handle[i].disk;
1337 705 : node = disk->filelist;
1338 2187947 : while (node) {
1339 : char path[PATH_MAX];
1340 : struct stat st;
1341 : struct snapraid_file* file;
1342 2186537 : int unsuccesful = 0;
1343 :
1344 2186537 : file = node->data;
1345 2186537 : node = node->next; /* next node */
1346 :
1347 : /* if not empty, it's already checked and continue to the next one */
1348 2186537 : if (file->size != 0) {
1349 4367549 : continue;
1350 : }
1351 :
1352 : /* if excluded continue to the next one */
1353 2834 : if (file_flag_has(file, FILE_IS_EXCLUDED)) {
1354 143 : continue;
1355 : }
1356 :
1357 : /* stat the file */
1358 2691 : pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
1359 2691 : ret = stat(path, &st);
1360 2691 : if (ret == -1) {
1361 163 : unsuccesful = 1;
1362 :
1363 163 : log_error("Error stating empty file '%s'. %s.\n", path, strerror(errno));
1364 163 : log_tag("error:%s:%s: Empty file stat error\n", disk->name, esc(file->sub));
1365 163 : ++error;
1366 2528 : } else if (!S_ISREG(st.st_mode)) {
1367 0 : unsuccesful = 1;
1368 :
1369 0 : log_tag("error:%s:%s: Empty file error for not regular file\n", disk->name, esc(file->sub));
1370 0 : ++error;
1371 2528 : } else if (st.st_size != 0) {
1372 14 : unsuccesful = 1;
1373 :
1374 14 : log_tag("error:%s:%s: Empty file error for size '%" PRIu64 "'\n", disk->name, esc(file->sub), st.st_size);
1375 14 : ++error;
1376 : }
1377 :
1378 2691 : if (fix && unsuccesful) {
1379 : int f;
1380 :
1381 : /* create the ancestor directories */
1382 154 : ret = mkancestor(path);
1383 154 : if (ret != 0) {
1384 : /* LCOV_EXCL_START */
1385 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1386 : log_fatal("Stopping\n");
1387 : ++unrecoverable_error;
1388 : goto bail;
1389 : /* LCOV_EXCL_STOP */
1390 : }
1391 :
1392 : /* create it */
1393 : /* O_NOFOLLOW: do not follow links to ensure to open the real file */
1394 154 : f = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_NOFOLLOW, 0600);
1395 154 : if (f == -1) {
1396 : /* LCOV_EXCL_START */
1397 : log_fatal("Error creating empty file '%s'. %s.\n", path, strerror(errno));
1398 : if (errno == EACCES) {
1399 : log_fatal("WARNING! Please give write permission to the file.\n");
1400 : } else {
1401 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1402 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1403 : }
1404 : log_fatal("Stopping\n");
1405 : ++unrecoverable_error;
1406 : goto bail;
1407 : /* LCOV_EXCL_STOP */
1408 : }
1409 :
1410 : /* set the original modification time */
1411 154 : ret = fmtime(f, file->mtime_sec, file->mtime_nsec);
1412 154 : if (ret != 0) {
1413 : /* LCOV_EXCL_START */
1414 : close(f);
1415 :
1416 : log_fatal("Error timing file '%s'. %s.\n", file->sub, strerror(errno));
1417 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1418 : log_fatal("Stopping\n");
1419 : ++unrecoverable_error;
1420 : goto bail;
1421 : /* LCOV_EXCL_STOP */
1422 : }
1423 :
1424 : /* close it */
1425 154 : ret = close(f);
1426 154 : if (ret != 0) {
1427 : /* LCOV_EXCL_START */
1428 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1429 : log_fatal("Stopping\n");
1430 : ++unrecoverable_error;
1431 : goto bail;
1432 : /* LCOV_EXCL_STOP */
1433 : }
1434 :
1435 154 : log_tag("fixed:%s:%s: Fixed empty file\n", disk->name, esc(file->sub));
1436 154 : ++recovered_error;
1437 :
1438 154 : log_tag("status:recovered:%s:%s\n", disk->name, esc(file->sub));
1439 154 : msg_info("recovered %s%s\n", disk->dir, file->sub);
1440 : }
1441 : }
1442 :
1443 : /* for each link in the disk */
1444 705 : disk = handle[i].disk;
1445 705 : node = disk->linklist;
1446 71529 : while (node) {
1447 : char path[PATH_MAX];
1448 : char pathto[PATH_MAX];
1449 : char linkto[PATH_MAX];
1450 : struct stat st;
1451 : struct stat stto;
1452 : struct snapraid_link* link;
1453 70119 : int unsuccesful = 0;
1454 70119 : int unrecoverable = 0;
1455 :
1456 70119 : link = node->data;
1457 70119 : node = node->next; /* next node */
1458 :
1459 : /* if excluded continue to the next one */
1460 70119 : if (link_flag_has(link, FILE_IS_EXCLUDED)) {
1461 2841 : continue;
1462 : }
1463 :
1464 67278 : if (link_flag_has(link, FILE_IS_HARDLINK)) {
1465 : /* stat the link */
1466 636 : pathprint(path, sizeof(path), "%s%s", disk->dir, link->sub);
1467 636 : ret = stat(path, &st);
1468 636 : if (ret == -1) {
1469 84 : unsuccesful = 1;
1470 :
1471 84 : log_error("Error stating hardlink '%s'. %s.\n", path, strerror(errno));
1472 84 : log_tag("hardlinkerror:%s:%s:%s: Hardlink stat error\n", disk->name, esc(link->sub), esc(link->linkto));
1473 84 : ++error;
1474 552 : } else if (!S_ISREG(st.st_mode)) {
1475 0 : unsuccesful = 1;
1476 :
1477 0 : log_tag("hardlinkerror:%s:%s:%s: Hardlink error for not regular file\n", disk->name, esc(link->sub), esc(link->linkto));
1478 0 : ++error;
1479 : }
1480 :
1481 : /* stat the "to" file */
1482 636 : pathprint(pathto, sizeof(pathto), "%s%s", disk->dir, link->linkto);
1483 636 : ret = stat(pathto, &stto);
1484 636 : if (ret == -1) {
1485 52 : unsuccesful = 1;
1486 :
1487 52 : if (errno == ENOENT) {
1488 52 : unrecoverable = 1;
1489 52 : if (fix) {
1490 : /* if the target doesn't exist, it's unrecoverable */
1491 : /* because we cannot create an hardlink of a file that */
1492 : /* doesn't exists */
1493 26 : ++unrecoverable_error;
1494 : } else {
1495 : /* but in check, we can assume that fixing will recover */
1496 : /* such missing file, so we assume a less drastic error */
1497 26 : ++error;
1498 : }
1499 : }
1500 :
1501 52 : log_error("Error stating hardlink-to '%s'. %s.\n", pathto, strerror(errno));
1502 52 : log_tag("hardlinkerror:%s:%s:%s: Hardlink to stat error\n", disk->name, esc(link->sub), esc(link->linkto));
1503 52 : ++error;
1504 584 : } else if (!S_ISREG(stto.st_mode)) {
1505 0 : unsuccesful = 1;
1506 :
1507 0 : log_tag("hardlinkerror:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, esc(link->sub), esc(link->linkto));
1508 0 : ++error;
1509 584 : } else if (!unsuccesful && st.st_ino != stto.st_ino) {
1510 0 : unsuccesful = 1;
1511 :
1512 0 : log_error("Mismatch hardlink '%s' and '%s'. Different inode.\n", path, pathto);
1513 0 : log_tag("hardlinkerror:%s:%s:%s: Hardlink mismatch for different inode\n", disk->name, esc(link->sub), esc(link->linkto));
1514 0 : ++error;
1515 : }
1516 : } else {
1517 : /* read the symlink */
1518 66642 : pathprint(path, sizeof(path), "%s%s", disk->dir, link->sub);
1519 66642 : ret = readlink(path, linkto, sizeof(linkto));
1520 66642 : if (ret < 0) {
1521 4333 : unsuccesful = 1;
1522 :
1523 4333 : log_error("Error reading symlink '%s'. %s.\n", path, strerror(errno));
1524 4333 : log_tag("symlinkerror:%s:%s: Symlink read error\n", disk->name, esc(link->sub));
1525 4333 : ++error;
1526 62309 : } else if (ret >= PATH_MAX) {
1527 0 : unsuccesful = 1;
1528 :
1529 0 : log_error("Error reading symlink '%s'. Symlink too long.\n", path);
1530 0 : log_tag("symlinkerror:%s:%s: Symlink read error\n", disk->name, esc(link->sub));
1531 0 : ++error;
1532 : } else {
1533 62309 : linkto[ret] = 0;
1534 :
1535 62309 : if (strcmp(linkto, link->linkto) != 0) {
1536 491 : unsuccesful = 1;
1537 :
1538 491 : log_tag("symlinkerror:%s:%s: Symlink data error '%s' instead of '%s'\n", disk->name, esc(link->sub), linkto, link->linkto);
1539 491 : ++error;
1540 : }
1541 : }
1542 : }
1543 :
1544 67278 : if (fix && unsuccesful && !unrecoverable) {
1545 : /* create the ancestor directories */
1546 3950 : ret = mkancestor(path);
1547 3950 : if (ret != 0) {
1548 : /* LCOV_EXCL_START */
1549 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1550 : log_fatal("Stopping\n");
1551 : ++unrecoverable_error;
1552 : goto bail;
1553 : /* LCOV_EXCL_STOP */
1554 : }
1555 :
1556 : /* if it exists, it must be deleted before recreating */
1557 3950 : ret = remove(path);
1558 3950 : if (ret != 0 && errno != ENOENT) {
1559 : /* LCOV_EXCL_START */
1560 : log_fatal("Error removing '%s'. %s.\n", path, strerror(errno));
1561 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1562 : log_fatal("Stopping\n");
1563 : ++unrecoverable_error;
1564 : goto bail;
1565 : /* LCOV_EXCL_STOP */
1566 : }
1567 :
1568 : /* create it */
1569 3950 : if (link_flag_has(link, FILE_IS_HARDLINK)) {
1570 32 : ret = hardlink(pathto, path);
1571 32 : if (ret != 0) {
1572 : /* LCOV_EXCL_START */
1573 : log_fatal("Error writing hardlink '%s' to '%s'. %s.\n", path, pathto, strerror(errno));
1574 : if (errno == EACCES) {
1575 : log_fatal("WARNING! Please give write permission to the hardlink.\n");
1576 : } else {
1577 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1578 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1579 : }
1580 : log_fatal("Stopping\n");
1581 : ++unrecoverable_error;
1582 : goto bail;
1583 : /* LCOV_EXCL_STOP */
1584 : }
1585 :
1586 32 : log_tag("hardlinkfixed:%s:%s: Fixed hardlink error\n", disk->name, esc(link->sub));
1587 32 : ++recovered_error;
1588 : } else {
1589 3918 : ret = symlink(link->linkto, path);
1590 3918 : if (ret != 0) {
1591 : /* LCOV_EXCL_START */
1592 : log_fatal("Error writing symlink '%s' to '%s'. %s.\n", path, link->linkto, strerror(errno));
1593 : if (errno == EACCES) {
1594 : log_fatal("WARNING! Please give write permission to the symlink.\n");
1595 : } else {
1596 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1597 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1598 : }
1599 : log_fatal("Stopping\n");
1600 : ++unrecoverable_error;
1601 : goto bail;
1602 : /* LCOV_EXCL_STOP */
1603 : }
1604 :
1605 3918 : log_tag("symlinkfixed:%s:%s: Fixed symlink error\n", disk->name, esc(link->sub));
1606 3918 : ++recovered_error;
1607 : }
1608 :
1609 3950 : log_tag("status:recovered:%s:%s\n", disk->name, esc(link->sub));
1610 3950 : msg_info("recovered %s%s\n", disk->dir, link->sub);
1611 : }
1612 : }
1613 :
1614 : /* for each dir in the disk */
1615 705 : disk = handle[i].disk;
1616 705 : node = disk->dirlist;
1617 1850 : while (node) {
1618 : char path[PATH_MAX];
1619 : struct stat st;
1620 : struct snapraid_dir* dir;
1621 440 : int unsuccesful = 0;
1622 :
1623 440 : dir = node->data;
1624 440 : node = node->next; /* next node */
1625 :
1626 : /* if excluded continue to the next one */
1627 440 : if (dir_flag_has(dir, FILE_IS_EXCLUDED)) {
1628 8 : continue;
1629 : }
1630 :
1631 : /* stat the dir */
1632 432 : pathprint(path, sizeof(path), "%s%s", disk->dir, dir->sub);
1633 432 : ret = stat(path, &st);
1634 432 : if (ret == -1) {
1635 23 : unsuccesful = 1;
1636 :
1637 23 : log_error("Error stating dir '%s'. %s.\n", path, strerror(errno));
1638 23 : log_tag("dir_error:%s:%s: Dir stat error\n", disk->name, esc(dir->sub));
1639 23 : ++error;
1640 409 : } else if (!S_ISDIR(st.st_mode)) {
1641 0 : unsuccesful = 1;
1642 :
1643 0 : log_tag("dir_error:%s:%s: Dir error for not directory\n", disk->name, esc(dir->sub));
1644 0 : ++error;
1645 : }
1646 :
1647 432 : if (fix && unsuccesful) {
1648 : /* create the ancestor directories */
1649 21 : ret = mkancestor(path);
1650 21 : if (ret != 0) {
1651 : /* LCOV_EXCL_START */
1652 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1653 : log_fatal("Stopping\n");
1654 : ++unrecoverable_error;
1655 : goto bail;
1656 : /* LCOV_EXCL_STOP */
1657 : }
1658 :
1659 : /* create it */
1660 21 : ret = mkdir(path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
1661 21 : if (ret != 0) {
1662 : /* LCOV_EXCL_START */
1663 : log_fatal("Error creating dir '%s'. %s.\n", path, strerror(errno));
1664 : if (errno == EACCES) {
1665 : log_fatal("WARNING! Please give write permission to the dir.\n");
1666 : } else {
1667 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1668 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1669 : }
1670 : log_fatal("Stopping\n");
1671 : ++unrecoverable_error;
1672 : goto bail;
1673 : /* LCOV_EXCL_STOP */
1674 : }
1675 :
1676 21 : log_tag("dir_fixed:%s:%s: Fixed dir error\n", disk->name, esc(dir->sub));
1677 21 : ++recovered_error;
1678 :
1679 21 : log_tag("status:recovered:%s:%s\n", disk->name, esc(dir->sub));
1680 21 : msg_info("recovered %s%s\n", disk->dir, dir->sub);
1681 : }
1682 : }
1683 : }
1684 :
1685 118 : state_progress_end(state, countpos, countmax, countsize);
1686 :
1687 : bail:
1688 : /* close all the files left open */
1689 826 : for (j = 0; j < diskmax; ++j) {
1690 708 : ret = handle_close(&handle[j]);
1691 708 : if (ret == -1) {
1692 : /* LCOV_EXCL_START */
1693 : log_fatal("DANGER! Unexpected close error in a data disk.\n");
1694 : ++unrecoverable_error;
1695 : /* continue, as we are already exiting */
1696 : /* LCOV_EXCL_STOP */
1697 : }
1698 : }
1699 :
1700 : /* remove all the files created from scratch that have not finished the processing */
1701 : /* it happens only when aborting pressing Ctrl+C or other reason. */
1702 118 : if (fix) {
1703 : /* for each disk */
1704 308 : for (i = 0; i < diskmax; ++i) {
1705 : tommy_node* node;
1706 : struct snapraid_disk* disk;
1707 :
1708 264 : if (!handle[i].disk)
1709 1 : continue;
1710 :
1711 : /* for each file in the disk */
1712 263 : disk = handle[i].disk;
1713 263 : node = disk->filelist;
1714 859122 : while (node) {
1715 : char path[PATH_MAX];
1716 : struct snapraid_file* file;
1717 :
1718 858596 : file = node->data;
1719 858596 : node = node->next; /* next node */
1720 :
1721 : /* if the file was not created, meaning that it was already existing */
1722 858596 : if (!file_flag_has(file, FILE_IS_CREATED)) {
1723 : /* nothing to do */
1724 1604218 : continue;
1725 : }
1726 :
1727 : /* if processing was finished */
1728 112974 : if (file_flag_has(file, FILE_IS_FINISHED)) {
1729 : /* nothing to do */
1730 112974 : continue;
1731 : }
1732 :
1733 : /* if the file was originally missing, and processing not yet finished */
1734 : /* we have to throw it away to ensure that at the next run we will retry */
1735 : /* to fix it, in case we select to undelete missing files */
1736 0 : pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
1737 :
1738 0 : ret = remove(path);
1739 0 : if (ret != 0) {
1740 : /* LCOV_EXCL_START */
1741 : log_fatal("Error removing '%s'. %s.\n", path, strerror(errno));
1742 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1743 : ++unrecoverable_error;
1744 : /* continue, as we are already exiting */
1745 : /* LCOV_EXCL_STOP */
1746 : }
1747 : }
1748 : }
1749 : }
1750 :
1751 118 : if (error || recovered_error || unrecoverable_error) {
1752 67 : msg_status("\n");
1753 67 : msg_status("%8u errors\n", error);
1754 67 : if (fix) {
1755 43 : msg_status("%8u recovered errors\n", recovered_error);
1756 : }
1757 134 : if (unrecoverable_error) {
1758 13 : msg_status("%8u UNRECOVERABLE errors\n", unrecoverable_error);
1759 : } else {
1760 : /* without checking, we don't know if they are really recoverable or not */
1761 54 : if (!state->opt.auditonly)
1762 52 : msg_status("%8u unrecoverable errors\n", unrecoverable_error);
1763 54 : if (fix)
1764 34 : msg_status("Everything OK\n");
1765 : }
1766 : } else {
1767 51 : msg_status("Everything OK\n");
1768 : }
1769 :
1770 118 : if (error && !fix)
1771 24 : log_fatal("WARNING! There are errors!\n");
1772 118 : if (unrecoverable_error)
1773 13 : log_fatal("DANGER! There are unrecoverable errors!\n");
1774 :
1775 118 : log_tag("summary:error:%u\n", error);
1776 118 : if (fix)
1777 44 : log_tag("summary:error_recovered:%u\n", recovered_error);
1778 118 : if (!state->opt.auditonly)
1779 113 : log_tag("summary:error_unrecoverable:%u\n", unrecoverable_error);
1780 118 : if (fix) {
1781 44 : if (error + recovered_error + unrecoverable_error == 0)
1782 1 : log_tag("summary:exit:ok\n");
1783 43 : else if (unrecoverable_error == 0)
1784 34 : log_tag("summary:exit:recovered\n");
1785 : else
1786 9 : log_tag("summary:exit:unrecoverable\n");
1787 74 : } else if (!state->opt.auditonly) {
1788 69 : if (error + unrecoverable_error == 0)
1789 47 : log_tag("summary:exit:ok\n");
1790 22 : else if (unrecoverable_error == 0)
1791 18 : log_tag("summary:exit:recoverable\n");
1792 : else
1793 4 : log_tag("summary:exit:unrecoverable\n");
1794 : } else { /* audit only */
1795 5 : if (error == 0)
1796 3 : log_tag("summary:exit:ok\n");
1797 : else
1798 2 : log_tag("summary:exit:error\n");
1799 : }
1800 118 : log_flush();
1801 :
1802 118 : free(failed);
1803 118 : free(failed_map);
1804 118 : free(handle);
1805 118 : free(buffer_alloc);
1806 118 : free(buffer);
1807 :
1808 : /* fails if some error are present after the run */
1809 118 : if (fix) {
1810 44 : if (state->opt.expect_unrecoverable) {
1811 9 : if (unrecoverable_error == 0)
1812 0 : return -1;
1813 : } else {
1814 35 : if (unrecoverable_error != 0)
1815 0 : return -1;
1816 : }
1817 : } else {
1818 74 : if (state->opt.expect_unrecoverable) {
1819 4 : if (unrecoverable_error == 0)
1820 0 : return -1;
1821 70 : } else if (state->opt.expect_recoverable) {
1822 20 : if (unrecoverable_error != 0 || error == 0)
1823 0 : return -1;
1824 : } else {
1825 50 : if (error != 0 || unrecoverable_error != 0)
1826 0 : return -1;
1827 : }
1828 : }
1829 :
1830 118 : return 0;
1831 : }
1832 :
1833 120 : int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount)
1834 : {
1835 : block_off_t blockmax;
1836 : data_off_t size;
1837 : data_off_t out_size;
1838 : int ret;
1839 : struct snapraid_parity_handle parity[LEV_MAX];
1840 : struct snapraid_parity_handle* parity_ptr[LEV_MAX];
1841 : unsigned error;
1842 : unsigned l;
1843 :
1844 120 : msg_progress("Initializing...\n");
1845 :
1846 120 : blockmax = parity_allocated_size(state);
1847 120 : size = blockmax * (data_off_t)state->block_size;
1848 :
1849 120 : if (blockstart > blockmax) {
1850 : /* LCOV_EXCL_START */
1851 : log_fatal("Error in the specified starting block %u. It's bigger than the parity size %u.\n", blockstart, blockmax);
1852 : exit(EXIT_FAILURE);
1853 : /* LCOV_EXCL_STOP */
1854 : }
1855 :
1856 : /* adjust the number of block to process */
1857 120 : if (blockcount != 0 && blockstart + blockcount < blockmax) {
1858 0 : blockmax = blockstart + blockcount;
1859 : }
1860 :
1861 120 : if (fix) {
1862 : /* if fixing, create the file and open for writing */
1863 : /* if it fails, we cannot continue */
1864 173 : for (l = 0; l < state->level; ++l) {
1865 129 : parity_ptr[l] = &parity[l];
1866 129 : ret = parity_create(parity_ptr[l], state->parity[l].path, &out_size, state->file_mode);
1867 129 : if (ret == -1) {
1868 : /* LCOV_EXCL_START */
1869 : log_fatal("WARNING! Without an accessible %s file, it isn't possible to fix any error.\n", lev_name(l));
1870 : exit(EXIT_FAILURE);
1871 : /* LCOV_EXCL_STOP */
1872 : }
1873 :
1874 129 : ret = parity_chsize(parity_ptr[l], size, &out_size, state->opt.skip_fallocate);
1875 129 : if (ret == -1) {
1876 : /* LCOV_EXCL_START */
1877 : log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
1878 : exit(EXIT_FAILURE);
1879 : /* LCOV_EXCL_STOP */
1880 : }
1881 : }
1882 76 : } else if (!state->opt.auditonly) {
1883 : /* if checking, open the file for reading */
1884 : /* it may fail if the file doesn't exist, in this case we continue to check the files */
1885 395 : for (l = 0; l < state->level; ++l) {
1886 325 : parity_ptr[l] = &parity[l];
1887 325 : ret = parity_open(parity_ptr[l], state->parity[l].path, state->file_mode);
1888 325 : if (ret == -1) {
1889 1 : msg_status("No accessible %s file, only files will be checked.\n", lev_name(l));
1890 : /* continue anyway */
1891 1 : parity_ptr[l] = 0;
1892 : }
1893 : }
1894 : } else {
1895 : /* otherwise don't use any parity */
1896 28 : for (l = 0; l < state->level; ++l)
1897 22 : parity_ptr[l] = 0;
1898 : }
1899 :
1900 120 : if (fix)
1901 44 : msg_progress("Fixing...\n");
1902 76 : else if (!state->opt.auditonly)
1903 70 : msg_progress("Checking...\n");
1904 : else
1905 6 : msg_progress("Hashing...\n");
1906 :
1907 120 : error = 0;
1908 :
1909 : /* skip degenerated cases of empty parity, or skipping all */
1910 120 : if (blockstart < blockmax) {
1911 118 : ret = state_check_process(state, fix, parity_ptr, blockstart, blockmax);
1912 118 : if (ret == -1) {
1913 : /* LCOV_EXCL_START */
1914 : ++error;
1915 : /* continue, as we are already exiting */
1916 : /* LCOV_EXCL_STOP */
1917 : }
1918 : }
1919 :
1920 : /* try to close only if opened */
1921 596 : for (l = 0; l < state->level; ++l) {
1922 476 : if (parity_ptr[l]) {
1923 453 : ret = parity_close(parity_ptr[l]);
1924 : /* LCOV_EXCL_START */
1925 : if (ret == -1) {
1926 : log_fatal("DANGER! Unexpected close error in %s disk.\n", lev_name(l));
1927 : ++error;
1928 : /* continue, as we are already exiting */
1929 : }
1930 : /* LCOV_EXCL_STOP */
1931 : }
1932 : }
1933 :
1934 : /* abort if error are present */
1935 120 : if (error != 0)
1936 0 : return -1;
1937 120 : return 0;
1938 : }
1939 :
|