Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "support.h"
21 : #include "util.h"
22 : #include "elem.h"
23 : #include "import.h"
24 : #include "search.h"
25 : #include "state.h"
26 : #include "parity.h"
27 : #include "handle.h"
28 : #include "raid/raid.h"
29 : #include "raid/combo.h"
30 :
31 : /****************************************************************************/
32 : /* check */
33 :
34 : /**
35 : * A block that failed the hash check, or that was deleted.
36 : */
37 : struct failed_struct {
38 : /**
39 : * If we know for sure that the block is garbage or missing
40 : * and it needs to be recovered and rewritten to the disk.
41 : */
42 : int is_bad;
43 :
44 : /**
45 : * If that we have recovered may be not updated data,
46 : * an old version, or just garbage.
47 : *
48 : * Essentially, it means that we are not sure what we have recovered
49 : * is really correct. It's just our best guess.
50 : *
51 : * These "recovered" block are also written to the disk if the block is marked as ::is_bad.
52 : * But these files are marked also as FILE_IS_DAMAGED, and then renamed to .unrecoverable.
53 : *
54 : * Note that this could happen only for CHG blocks.
55 : */
56 : int is_outofdate;
57 :
58 : unsigned index; /**< Index of the failed block. */
59 : struct snapraid_block* block; /**< The failed block */
60 : struct snapraid_disk* disk; /**< The failed disk. */
61 : struct snapraid_file* file; /**< The failed file. 0 for DELETED block. */
62 : block_off_t file_pos; /**< Offset inside the file */
63 : struct snapraid_handle* handle; /**< The handle containing the failed block, or 0 for a DELETED block */
64 : };
65 :
66 : /**
67 : * Check if a block hash matches the specified buffer.
68 : * Return ==0 if equal
69 : */
70 473119 : static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_block* block, unsigned pos_size, unsigned char* buffer, unsigned char* buffer_zero)
71 : {
72 : unsigned char hash[HASH_MAX];
73 :
74 : /* now compute the hash of the valid part */
75 473119 : if (rehash) {
76 0 : memhash(state->prevhash, state->prevhashseed, hash, buffer, pos_size);
77 : } else {
78 473119 : memhash(state->hash, state->hashseed, hash, buffer, pos_size);
79 : }
80 :
81 : /* compare the hash */
82 473119 : if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) {
83 19028 : return -1;
84 : }
85 :
86 : /* compare to the end of the block */
87 454091 : if (pos_size < state->block_size) {
88 187372 : if (memcmp(buffer + pos_size, buffer_zero + pos_size, state->block_size - pos_size) != 0) {
89 54 : return -1;
90 : }
91 : }
92 :
93 454037 : return 0;
94 : }
95 :
96 : /**
97 : * Check if the hash of all the failed block we are expecting to recover are now matching.
98 : */
99 214644 : static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void* buffer_zero)
100 : {
101 : unsigned j;
102 : int hash_checked;
103 :
104 214644 : hash_checked = 0; /* keep track if we check at least one block */
105 :
106 : /* check if the recovered blocks are OK */
107 673203 : for (j = 0; j < failed_count; ++j) {
108 : /* if we are expected to recover this block */
109 477641 : if (!failed[failed_map[j]].is_outofdate
110 : /* if the block has a hash to check */
111 473119 : && block_has_updated_hash(failed[failed_map[j]].block)
112 : ) {
113 : /* if a hash doesn't match, fail the check */
114 473119 : unsigned pos_size = file_block_size(failed[failed_map[j]].file, failed[failed_map[j]].file_pos, state->block_size);
115 473119 : if (blockcmp(state, rehash, failed[failed_map[j]].block, pos_size, buffer[failed[failed_map[j]].index], buffer_zero) != 0) {
116 19082 : log_tag("hash_error: Hash mismatch on entry %u\n", failed_map[j]);
117 19082 : return 0;
118 : }
119 :
120 454037 : hash_checked = 1;
121 : }
122 : }
123 :
124 : /* if nothing checked, we reject it */
125 : /* note that we are excluding this case at upper level */
126 : /* but checking again doesn't hurt */
127 195562 : if (!hash_checked) {
128 : /* LCOV_EXCL_START */
129 : return 0;
130 : /* LCOV_EXCL_STOP */
131 : }
132 :
133 : /* if we checked something, and no block failed the check */
134 : /* recompute all the redundancy information */
135 195562 : raid_gen(diskmax, state->level, state->block_size, buffer);
136 195562 : return 1;
137 : }
138 :
139 : /**
140 : * Check if specified parity is now matching with a recomputed one.
141 : */
142 8 : static int is_parity_matching(struct snapraid_state* state, unsigned diskmax, unsigned i, void** buffer, void** buffer_recov)
143 : {
144 : /* recompute parity, note that we don't need parity over i */
145 8 : raid_gen(diskmax, i + 1, state->block_size, buffer);
146 :
147 : /* if the recovered parity block matches */
148 8 : if (memcmp(buffer[diskmax + i], buffer_recov[i], state->block_size) == 0) {
149 : /* recompute all the redundancy information */
150 8 : raid_gen(diskmax, state->level, state->block_size, buffer);
151 8 : return 1;
152 : }
153 :
154 0 : return 0;
155 : }
156 :
157 : /**
158 : * Repair errors.
159 : * Return <0 if failure for missing strategy, >0 if data is wrong and we cannot rebuild correctly, 0 on success.
160 : * If success, the parity are computed in the buffer variable.
161 : */
162 252549 : static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void** buffer_recov, void* buffer_zero)
163 : {
164 : unsigned i, n;
165 : int error;
166 : int has_hash;
167 : int id[LEV_MAX];
168 : int ip[LEV_MAX];
169 :
170 : /* no fix required, already checked at higher level, but just to be sure */
171 252549 : if (failed_count == 0) {
172 : /* LCOV_EXCL_START */
173 : /* recompute only the parity */
174 : raid_gen(diskmax, state->level, state->block_size, buffer);
175 : return 0;
176 : /* LCOV_EXCL_STOP */
177 : }
178 :
179 252549 : n = state->level;
180 252549 : error = 0;
181 :
182 : /* setup vector of failed disk indexes */
183 891351 : for (i = 0; i < failed_count; ++i)
184 638802 : id[i] = failed[failed_map[i]].index;
185 :
186 : /* check if there is at least a failed block that can be checked for correctness using the hash */
187 : /* if there isn't, we have to sacrifice a parity block to check that the result is correct */
188 252549 : has_hash = 0;
189 891351 : for (i = 0; i < failed_count; ++i) {
190 : /* if we are expected to recover this block */
191 638802 : if (!failed[failed_map[i]].is_outofdate
192 : /* if the block has a hash to check */
193 629758 : && block_has_updated_hash(failed[failed_map[i]].block)
194 : )
195 629750 : has_hash = 1;
196 : }
197 :
198 : /* if we don't have a hash, but we have an extra parity */
199 : /* (strictly-less failures than number of parities) */
200 252549 : if (!has_hash && failed_count < n) {
201 : /* number of parity to use, one more to check the recovering */
202 8 : unsigned r = failed_count + 1;
203 :
204 : /* all combinations (r of n) parities */
205 8 : combination_first(r, n, ip);
206 : do {
207 : /* if a parity is missing, do nothing */
208 24 : for (i = 0; i < r; ++i) {
209 16 : if (buffer_recov[ip[i]] == 0)
210 0 : break;
211 : }
212 8 : if (i != r)
213 0 : continue;
214 :
215 : /* copy the parities to use, one less because the last is used for checking */
216 16 : for (i = 0; i < r - 1; ++i)
217 8 : memcpy(buffer[diskmax + ip[i]], buffer_recov[ip[i]], state->block_size);
218 :
219 : /* recover using one less parity, the ip[r-1] one */
220 8 : raid_data(r - 1, id, ip, diskmax, state->block_size, buffer);
221 :
222 : /* use the remaining ip[r-1] parity to check the result */
223 8 : if (is_parity_matching(state, diskmax, ip[r - 1], buffer, buffer_recov))
224 8 : return 0;
225 :
226 : /* log */
227 0 : log_tag("parity_error:%u:", pos);
228 0 : for (i = 0; i < r; ++i) {
229 0 : if (i != 0)
230 0 : log_tag("/");
231 0 : log_tag("%s", lev_config_name(ip[i]));
232 : }
233 0 : log_tag(":parity: Parity mismatch\n");
234 0 : ++error;
235 0 : } while (combination_next(r, n, ip));
236 : }
237 :
238 : /* if we have a hash, and enough parities */
239 : /* (less-or-equal failures than number of parities) */
240 252541 : if (has_hash && failed_count <= n) {
241 : /* number of parities to use equal at the number of failures */
242 204895 : unsigned r = failed_count;
243 :
244 : /* all combinations (r of n) parities */
245 204895 : combination_first(r, n, ip);
246 : do {
247 : /* if a parity is missing, do nothing */
248 692285 : for (i = 0; i < r; ++i) {
249 477641 : if (buffer_recov[ip[i]] == 0)
250 0 : break;
251 : }
252 214644 : if (i != r)
253 0 : continue;
254 :
255 : /* copy the parities to use */
256 692285 : for (i = 0; i < r; ++i)
257 477641 : memcpy(buffer[diskmax + ip[i]], buffer_recov[ip[i]], state->block_size);
258 :
259 : /* recover */
260 214644 : raid_data(r, id, ip, diskmax, state->block_size, buffer);
261 :
262 : /* use the hash to check the result */
263 214644 : if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero))
264 195562 : return 0;
265 :
266 : /* log */
267 19082 : log_tag("parity_error:%u:", pos);
268 38164 : for (i = 0; i < r; ++i) {
269 19082 : if (i != 0)
270 0 : log_tag("/");
271 19082 : log_tag("%s", lev_config_name(ip[i]));
272 : }
273 19082 : log_tag(":hash: Hash mismatch\n");
274 19082 : ++error;
275 38164 : } while (combination_next(r, n, ip));
276 : }
277 :
278 : /* return the number of failed attempts, or -1 if no strategy */
279 56979 : if (error)
280 9333 : return error;
281 :
282 47646 : log_tag("strategy_error:%u: No strategy to recover from %u failures with %u parity %s hash\n",
283 : pos, failed_count, n, has_hash ? "with" : "without");
284 47646 : return -1;
285 : }
286 :
287 870958 : static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void** buffer_recov, void* buffer_zero)
288 : {
289 : int ret;
290 : int error;
291 : unsigned j;
292 : int n;
293 : int something_to_recover;
294 : int something_unsynced;
295 : char esc_buffer[ESC_MAX];
296 :
297 870958 : error = 0;
298 :
299 : /* if nothing failed, just recompute the parity */
300 870958 : if (failed_count == 0) {
301 583497 : raid_gen(diskmax, state->level, state->block_size, buffer);
302 583497 : return 0;
303 : }
304 :
305 : /* logs the status */
306 1016282 : for (j = 0; j < failed_count; ++j) {
307 : const char* desc;
308 : const char* hash;
309 : const char* data;
310 728821 : struct snapraid_block* block = failed[j].block;
311 728821 : unsigned block_state = block_state_get(block);
312 :
313 728821 : switch (block_state) {
314 18454 : case BLOCK_STATE_DELETED : desc = "delete"; break;
315 13851 : case BLOCK_STATE_CHG : desc = "change"; break;
316 62282 : case BLOCK_STATE_REP : desc = "replace"; break;
317 634234 : case BLOCK_STATE_BLK : desc = "block"; break;
318 : /* LCOV_EXCL_START */
319 : default : desc = "unknown"; break;
320 : /* LCOV_EXCL_STOP */
321 : }
322 :
323 728821 : if (hash_is_invalid(block->hash)) {
324 13843 : hash = "lost";
325 714978 : } else if (hash_is_zero(block->hash)) {
326 8 : hash = "zero";
327 : } else {
328 714970 : hash = "known";
329 : }
330 :
331 728821 : if (failed[j].is_bad)
332 643096 : data = "bad";
333 : else
334 85725 : data = "good";
335 :
336 728821 : if (failed[j].file) {
337 710367 : struct snapraid_disk* disk = failed[j].disk;
338 710367 : struct snapraid_file* file = failed[j].file;
339 710367 : block_off_t file_pos = failed[j].file_pos;
340 :
341 710367 : log_tag("entry:%u:%s:%s:%s:%s:%s:%u:\n", j, desc, hash, data, disk->name, esc_tag(file->sub, esc_buffer), file_pos);
342 : } else {
343 18454 : log_tag("entry:%u:%s:%s:%s:\n", j, desc, hash, data);
344 : }
345 : }
346 :
347 : /* Here we have to try two different strategies to recover, because in case the 'sync' */
348 : /* process is aborted, we don't know if the parity data is really updated just like after 'sync', */
349 : /* or if it still represents the state before the 'sync'. */
350 :
351 : /* Note that if the 'sync' ends normally, we don't have any DELETED, REP and CHG blocks */
352 : /* and the two strategies are identical */
353 :
354 : /* As first, we assume that the parity IS updated for the current state */
355 : /* and that we are going to recover the state after the last 'sync'. */
356 : /* In this case, parity contains info from BLK, REP and CHG blocks, */
357 : /* but not for DELETED. */
358 : /* We need to put in the recovering process only the bad blocks, because all the */
359 : /* others already contains the correct data read from disk, and the parity is correctly computed for them. */
360 : /* We are interested to recover BLK, REP and CHG blocks if they are marked as bad, */
361 : /* but we are not interested in DELETED ones. */
362 :
363 287461 : n = 0;
364 287461 : something_to_recover = 0; /* keep track if there is at least one block to fix */
365 1016282 : for (j = 0; j < failed_count; ++j) {
366 728821 : if (failed[j].is_bad) {
367 643096 : unsigned block_state = block_state_get(failed[j].block);
368 :
369 643096 : assert(block_state != BLOCK_STATE_DELETED); /* we cannot have bad DELETED blocks */
370 :
371 : /* if we have the hash for it */
372 643096 : if ((block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP)
373 : /* try to fetch the block using the known hash */
374 643088 : && (state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0
375 638616 : || state_search_fetch(state, rehash, failed[j].file, failed[j].file_pos, failed[j].block, buffer[failed[j].index]) == 0)
376 : ) {
377 : /* we already have corrected it! */
378 22382 : log_tag("hash_import: Fixed entry %u\n", j);
379 : } else {
380 : /* otherwise try to recover it */
381 620714 : failed_map[n] = j;
382 620714 : ++n;
383 :
384 : /* we have something to try to recover */
385 620714 : something_to_recover = 1;
386 : }
387 : }
388 : }
389 :
390 : /* if nothing to fix */
391 287461 : if (!something_to_recover) {
392 43956 : log_tag("recover_sync:%u:%u: Skipped for already recovered\n", pos, n);
393 :
394 : /* recompute only the parity */
395 43956 : raid_gen(diskmax, state->level, state->block_size, buffer);
396 43956 : return 0;
397 : }
398 :
399 243505 : ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, buffer_zero);
400 243505 : if (ret == 0) {
401 : /* reprocess the CHG blocks, for which we don't have a hash to check */
402 : /* if they were BAD we have to use some heuristics to ensure that we have recovered */
403 : /* the state after the sync. If unsure, we assume the worst case */
404 :
405 667198 : for (j = 0; j < failed_count; ++j) {
406 : /* we take care only of BAD blocks we have to write back */
407 476150 : if (failed[j].is_bad) {
408 462678 : unsigned block_state = block_state_get(failed[j].block);
409 :
410 : /* BLK and REP blocks are always OK, because at this point */
411 : /* we have already checked their hash */
412 462678 : if (block_state != BLOCK_STATE_CHG) {
413 462670 : assert(block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP);
414 462670 : continue;
415 : }
416 :
417 : /* for CHG blocks we have to 'guess' if they are correct or not */
418 :
419 : /* if the hash is invalid we cannot check the result */
420 : /* this could happen if we have lost this information */
421 : /* after an aborted sync */
422 8 : if (hash_is_invalid(failed[j].block->hash)) {
423 : /* it may contain garbage */
424 0 : failed[j].is_outofdate = 1;
425 :
426 0 : log_tag("hash_unknown: Unknown hash on entry %u\n", j);
427 8 : } else if (hash_is_zero(failed[j].block->hash)) {
428 : /* if the block is not filled with 0, we are sure to have */
429 : /* restored it to the state after the 'sync' */
430 : /* instead, if the block is filled with 0, it could be either that the */
431 : /* block after the sync is really filled by 0, or that */
432 : /* we restored the block before the 'sync'. */
433 8 : if (memcmp(buffer[failed[j].index], buffer_zero, state->block_size) == 0) {
434 : /* it may contain garbage */
435 8 : failed[j].is_outofdate = 1;
436 :
437 8 : log_tag("hash_unknown: Maybe old zero on entry %u\n", j);
438 : }
439 : } else {
440 : /* if the hash is different than the previous one, we are sure to have */
441 : /* restored it to the state after the 'sync' */
442 : /* instead, if the hash matches, it could be either that the */
443 : /* block after the sync has this hash, or that */
444 : /* we restored the block before the 'sync'. */
445 0 : unsigned pos_size = file_block_size(failed[j].file, failed[j].file_pos, state->block_size);
446 0 : if (blockcmp(state, rehash, failed[j].block, pos_size, buffer[failed[j].index], buffer_zero) == 0) {
447 : /* it may contain garbage */
448 0 : failed[j].is_outofdate = 1;
449 :
450 0 : log_tag("hash_unknown: Maybe old data on entry %u\n", j);
451 : }
452 : }
453 : }
454 : }
455 :
456 191048 : return 0;
457 : }
458 52457 : if (ret > 0)
459 9333 : error += ret;
460 :
461 52457 : if (ret < 0)
462 43124 : log_tag("recover_sync:%u:%u: Failed with no attempts\n", pos, n);
463 : else
464 9333 : log_tag("recover_sync:%u:%u: Failed with %d attempts\n", pos, n, ret);
465 :
466 : /* Now assume that the parity IS NOT updated at the current state, */
467 : /* but still represent the state before the last 'sync' process. */
468 : /* In this case, parity contains info from BLK, REP (old version), CHG (old version) and DELETED blocks, */
469 : /* but not for REP (new version) and CHG (new version). */
470 : /* We are interested to recover BLK ones marked as bad, */
471 : /* but we are not interested to recover CHG (new version) and REP (new version) blocks, */
472 : /* even if marked as bad, because we don't have parity for them and it's just impossible, */
473 : /* and we are not interested to recover DELETED ones. */
474 52457 : n = 0;
475 52457 : something_to_recover = 0; /* keep track if there is at least one block to fix */
476 52457 : something_unsynced = 0; /* keep track if we have some unsynced info to process */
477 232692 : for (j = 0; j < failed_count; ++j) {
478 180235 : unsigned block_state = block_state_get(failed[j].block);
479 :
480 180235 : if (block_state == BLOCK_STATE_DELETED
481 171191 : || block_state == BLOCK_STATE_CHG
482 171191 : || block_state == BLOCK_STATE_REP
483 : ) {
484 : /* If the block is CHG, REP or DELETED, we don't have the original content of block, */
485 : /* and we must try to recover it. */
486 : /* This apply to CHG and REP blocks even if they are not marked bad, */
487 : /* because the parity is computed with old content, and not with the new one. */
488 : /* Note that this recovering is done just to make possible to recover any other BLK one, */
489 : /* we are not really interested in DELETED, CHG (old version) and REP (old version). */
490 9044 : something_unsynced = 1;
491 :
492 9044 : if (block_state == BLOCK_STATE_CHG
493 0 : && hash_is_zero(failed[j].block->hash)
494 : ) {
495 : /* If the block was a ZERO block, restore it to the original 0 as before the 'sync' */
496 : /* We do this to just allow recovering of other BLK ones */
497 :
498 0 : memset(buffer[failed[j].index], 0, state->block_size);
499 : /* note that from now the buffer is definitively lost */
500 : /* we can do this only because it's the last retry of recovering */
501 :
502 : /* try to fetch the old block using the old hash for CHG and DELETED blocks */
503 9044 : } else if ((block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_DELETED)
504 9044 : && hash_is_unique(failed[j].block->hash)
505 9044 : && state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0) {
506 :
507 : /* note that from now the buffer is definitively lost */
508 : /* we can do this only because it's the last retry of recovering */
509 : } else {
510 : /* otherwise try to recover it */
511 9044 : failed_map[n] = j;
512 9044 : ++n;
513 :
514 : /* note that we don't set something_to_recover, because we are */
515 : /* not really interested to recover *only* old blocks. */
516 : }
517 :
518 : /* avoid to use the hash of this block to verify the recovering */
519 : /* this applies to REP blocks because we are going to recover the old state */
520 : /* and the REP hash represent the new one */
521 : /* it also applies to CHG and DELETE blocks because we want to have */
522 : /* a successful recovering only if a BLK one is matching */
523 9044 : failed[j].is_outofdate = 1;
524 171191 : } else if (failed[j].is_bad) {
525 : /* If the block is bad we don't know its content, and we try to recover it */
526 : /* At this point, we can have only BLK ones */
527 :
528 171191 : assert(block_state == BLOCK_STATE_BLK);
529 :
530 : /* we have something we are interested to recover */
531 171191 : something_to_recover = 1;
532 :
533 : /* we try to recover it */
534 171191 : failed_map[n] = j;
535 171191 : ++n;
536 : }
537 : }
538 :
539 : /* if nothing to fix, we just don't try */
540 : /* if nothing unsynced we also don't retry, because it's the same try as before */
541 52457 : if (something_to_recover && something_unsynced) {
542 9044 : ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, buffer_zero);
543 9044 : if (ret == 0) {
544 : /* reprocess the REP and CHG blocks, for which we have recovered and old state */
545 : /* that we don't want to save into disk */
546 : /* we have already marked them, but we redo it for logging */
547 :
548 13566 : for (j = 0; j < failed_count; ++j) {
549 : /* we take care only of BAD blocks we have to write back */
550 9044 : if (failed[j].is_bad) {
551 4522 : unsigned block_state = block_state_get(failed[j].block);
552 :
553 4522 : if (block_state == BLOCK_STATE_CHG
554 4522 : || block_state == BLOCK_STATE_REP
555 : ) {
556 : /* mark that we have restored an old state */
557 : /* and we don't want to write it to the disk */
558 0 : failed[j].is_outofdate = 1;
559 :
560 0 : log_tag("hash_unknown: Surely old data on entry %u\n", j);
561 : }
562 : }
563 : }
564 :
565 4522 : return 0;
566 : }
567 4522 : if (ret > 0)
568 0 : error += ret;
569 :
570 9044 : if (ret < 0)
571 4522 : log_tag("recover_unsync:%u:%u: Failed with no attempts\n", pos, n);
572 : else
573 0 : log_tag("recover_unsync:%u:%u: Failed with %d attempts\n", pos, n, ret);
574 : } else {
575 43413 : log_tag("recover_unsync:%u:%u: Skipped for%s%s\n", pos, n,
576 : !something_to_recover ? " nothing to recover" : "",
577 : !something_unsynced ? " nothing unsynched" : ""
578 : );
579 : }
580 :
581 : /* return the number of failed attempts, or -1 if no strategy */
582 47935 : if (error)
583 4811 : return error;
584 : else
585 43124 : return -1;
586 : }
587 :
588 : /**
589 : * Post process all the files at the specified block index ::i.
590 : * For each file, if we are at the last block, closes it,
591 : * adjust the timestamp, and print the result.
592 : *
593 : * This works with the assumption to always process the whole files to
594 : * fix. This assumption is not always correct, and in such case we have to
595 : * skip the whole postprocessing. And example, is when fixing only bad blocks.
596 : */
597 935479 : static int file_post(struct snapraid_state* state, int fix, unsigned i, struct snapraid_handle* handle, unsigned diskmax)
598 : {
599 : unsigned j;
600 : int ret;
601 : char esc_buffer[ESC_MAX];
602 : char esc_buffer_alt[ESC_MAX];
603 :
604 : /* if we are processing only bad blocks, we don't have to do any post-processing */
605 : /* as we don't have any guarantee to process the last block of the fixed files */
606 935479 : if (state->opt.badonly)
607 9374 : return 0;
608 :
609 : /* for all the files print the final status, and does the final time fix */
610 : /* we also ensure to close files after processing the last block */
611 6482735 : for (j = 0; j < diskmax; ++j) {
612 : struct snapraid_block* block;
613 : struct snapraid_disk* disk;
614 : struct snapraid_file* collide_file;
615 : struct snapraid_file* file;
616 : block_off_t file_pos;
617 : char path[PATH_MAX];
618 : uint64_t inode;
619 :
620 5556630 : disk = handle[j].disk;
621 5556630 : if (!disk) {
622 : /* if no disk, nothing to do */
623 3408432 : continue;
624 : }
625 :
626 5542569 : block = fs_par2block_find(disk, i);
627 5542569 : if (!block_has_file(block)) {
628 : /* if no file, nothing to do */
629 213402 : continue;
630 : }
631 :
632 5329167 : file = fs_par2file_get(disk, i, &file_pos);
633 5329167 : pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
634 :
635 : /* if it isn't the last block in the file */
636 5329167 : if (!file_block_is_last(file, file_pos)) {
637 : /* nothing to do */
638 3166908 : continue;
639 : }
640 :
641 : /* if the file is excluded, we have nothing to adjust as the file is never written */
642 2162259 : if (file_flag_has(file, FILE_IS_EXCLUDED)
643 2065266 : || (state->opt.syncedonly && file_flag_has(file, FILE_IS_UNSYNCED))) {
644 : /* nothing to do, but close the file */
645 : goto close_and_continue;
646 : }
647 :
648 : /* finish the fix process if it's the last block of the files */
649 2065266 : if (fix) {
650 : /* mark that we finished with this file */
651 : /* to identify later any NOT finished ones */
652 757234 : file_flag_set(file, FILE_IS_FINISHED);
653 :
654 : /* if the file is damaged, meaning that a fix failed */
655 757234 : if (file_flag_has(file, FILE_IS_DAMAGED)) {
656 : /* rename it to .unrecoverable */
657 : char path_to[PATH_MAX];
658 :
659 63275 : pathprint(path_to, sizeof(path_to), "%s%s.unrecoverable", disk->dir, file->sub);
660 :
661 : /* ensure to close the file before renaming */
662 63275 : if (handle[j].file == file) {
663 63275 : ret = handle_close(&handle[j]);
664 63275 : if (ret != 0) {
665 : /* LCOV_EXCL_START */
666 : log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
667 : log_fatal("DANGER! Unexpected close error in a data disk.\n");
668 : return -1;
669 : /* LCOV_EXCL_STOP */
670 : }
671 : }
672 :
673 63275 : ret = rename(path, path_to);
674 63275 : if (ret != 0) {
675 : /* LCOV_EXCL_START */
676 : log_fatal("Error renaming '%s' to '%s'. %s.\n", path, path_to, strerror(errno));
677 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
678 : return -1;
679 : /* LCOV_EXCL_STOP */
680 : }
681 :
682 63275 : log_tag("status:unrecoverable:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
683 63275 : msg_info("unrecoverable %s\n", fmt_term(disk, file->sub, esc_buffer));
684 :
685 : /* and do not set the time if damaged */
686 63275 : goto close_and_continue;
687 : }
688 :
689 : /* if the file is not fixed, meaning that it is untouched */
690 693959 : if (!file_flag_has(file, FILE_IS_FIXED)) {
691 : /* nothing to do, but close the file */
692 574554 : goto close_and_continue;
693 : }
694 :
695 : /* if the file is closed or different than the one expected, reopen it */
696 : /* a different open file could happen when filtering for bad blocks */
697 119405 : if (handle[j].file != file) {
698 : /* close a potential different file */
699 0 : ret = handle_close(&handle[j]);
700 0 : if (ret != 0) {
701 : /* LCOV_EXCL_START */
702 : log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc_tag(handle[j].file->sub, esc_buffer), strerror(errno));
703 : log_fatal("DANGER! Unexpected close error in a data disk.\n");
704 : return -1;
705 : /* LCOV_EXCL_STOP */
706 : }
707 :
708 : /* reopen it as readonly, as to set the mtime readonly access it's enough */
709 : /* we know that the file exists because it has the FILE_IS_FIXED tag */
710 0 : ret = handle_open(&handle[j], file, state->file_mode, log_error, 0);
711 0 : if (ret != 0) {
712 : /* LCOV_EXCL_START */
713 : log_tag("error:%u:%s:%s: Open error. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
714 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
715 : return -1;
716 : /* LCOV_EXCL_STOP */
717 : }
718 : }
719 :
720 119405 : log_tag("status:recovered:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
721 119405 : msg_info("recovered %s\n", fmt_term(disk, file->sub, esc_buffer));
722 :
723 119405 : inode = handle[j].st.st_ino;
724 :
725 : /* search for the corresponding inode */
726 119405 : collide_file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
727 :
728 : /* if the inode is already in the database and it refers at a different file name, */
729 : /* we can fix the file time ONLY if the time and size allow to differentiate */
730 : /* between the two files */
731 :
732 : /* for example, suppose we delete a bunch of files with all the same size and time, */
733 : /* when recreating them the inodes may be reused in a different order, */
734 : /* and at the next sync some files may have matching inode/size/time even if different name */
735 : /* not allowing sync to detect that the file is changed and not renamed */
736 119405 : if (!collide_file /* if not in the database, there is no collision */
737 31334 : || strcmp(collide_file->sub, file->sub) == 0 /* if the name is the same, it's the right collision */
738 22534 : || collide_file->size != file->size /* if the size is different, the collision is identified */
739 2 : || collide_file->mtime_sec != file->mtime_sec /* if the mtime is different, the collision is identified */
740 1 : || collide_file->mtime_nsec != file->mtime_nsec /* same for mtime_nsec */
741 : ) {
742 : /* set the original modification time */
743 119405 : ret = handle_utime(&handle[j]);
744 238810 : if (ret == -1) {
745 : /* LCOV_EXCL_START */
746 : /* mark the file as damaged */
747 : file_flag_set(file, FILE_IS_DAMAGED);
748 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
749 : return -1;
750 : /* LCOV_EXCL_STOP */
751 : }
752 : } else {
753 0 : log_tag("collision:%s:%s:%s: Not setting modification time to avoid inode collision\n", disk->name, esc_tag(file->sub, esc_buffer), esc_tag(collide_file->sub, esc_buffer_alt));
754 : }
755 : } else {
756 : /* we are not fixing, but only checking */
757 : /* print just the final status */
758 1308032 : if (file_flag_has(file, FILE_IS_DAMAGED)) {
759 9420 : if (state->opt.auditonly) {
760 2798 : log_tag("status:damaged:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
761 2798 : msg_info("damaged %s\n", fmt_term(disk, file->sub, esc_buffer));
762 : } else {
763 6622 : log_tag("status:unrecoverable:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
764 6622 : msg_info("unrecoverable %s\n", fmt_term(disk, file->sub, esc_buffer));
765 : }
766 1298612 : } else if (file_flag_has(file, FILE_IS_FIXED)) {
767 76223 : log_tag("status:recoverable:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
768 76223 : msg_info("recoverable %s\n", fmt_term(disk, file->sub, esc_buffer));
769 : } else {
770 : /* we don't use msg_verbose() because it also goes into the log */
771 1222389 : if (msg_level >= MSG_VERBOSE) {
772 23229 : log_tag("status:correct:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
773 23229 : msg_info("correct %s\n", fmt_term(disk, file->sub, esc_buffer));
774 : }
775 : }
776 : }
777 :
778 : close_and_continue:
779 : /* if the opened file is the correct one, close it */
780 : /* in case of excluded and fragmented files it's possible */
781 : /* that the opened file is not the current one */
782 2162259 : if (handle[j].file == file) {
783 : /* ensure to close the file just after finishing with it */
784 : /* to avoid to keep it open without any possible use */
785 1978605 : ret = handle_close(&handle[j]);
786 1978605 : if (ret != 0) {
787 : /* LCOV_EXCL_START */
788 : log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
789 : log_fatal("DANGER! Unexpected close error in a data disk.\n");
790 : return -1;
791 : /* LCOV_EXCL_STOP */
792 : }
793 : }
794 : }
795 :
796 926105 : return 0;
797 : }
798 :
799 : /**
800 : * Check if we have to process the specified block index ::i.
801 : */
802 1870958 : static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct snapraid_handle* handle, unsigned diskmax)
803 : {
804 : snapraid_info info;
805 : unsigned j;
806 : unsigned l;
807 :
808 : /* get block specific info */
809 1870958 : info = info_get(&state->infoarr, i);
810 :
811 : /* if we filter for only bad blocks */
812 1870958 : if (state->opt.badonly) {
813 : /* skip if this is not bad */
814 18748 : if (!info_get_bad(info))
815 15204 : return 0;
816 : }
817 :
818 : /* for each parity */
819 2125818 : for (l = 0; l < state->level; ++l) {
820 2042438 : if (!state->parity[l].is_excluded) {
821 1772374 : return 1;
822 : }
823 : }
824 :
825 : /* for each disk */
826 386636 : for (j = 0; j < diskmax; ++j) {
827 : struct snapraid_block* block;
828 :
829 : /* if no disk, nothing to check */
830 351348 : if (!handle[j].disk)
831 0 : continue;
832 :
833 351348 : block = fs_par2block_find(handle[j].disk, i);
834 :
835 : /* try to recover all files, even the ones without hash */
836 : /* because in some cases we can recover also them */
837 351348 : if (block_has_file(block)) {
838 344686 : struct snapraid_file* file = fs_par2file_get(handle[j].disk, i, 0);
839 344686 : if (!file_flag_has(file, FILE_IS_EXCLUDED)) { /* only if the file is not filtered out */
840 48092 : return 1;
841 : }
842 : }
843 : }
844 :
845 35288 : return 0;
846 : }
847 :
848 118 : static int state_check_process(struct snapraid_state* state, int fix, struct snapraid_parity_handle** parity, block_off_t blockstart, block_off_t blockmax)
849 : {
850 : struct snapraid_handle* handle;
851 : unsigned diskmax;
852 : block_off_t i;
853 : unsigned j;
854 : void* buffer_alloc;
855 : void** buffer;
856 : unsigned buffermax;
857 : int ret;
858 : data_off_t countsize;
859 : block_off_t countpos;
860 : block_off_t countmax;
861 : unsigned error;
862 : unsigned unrecoverable_error;
863 : unsigned recovered_error;
864 : struct failed_struct* failed;
865 : unsigned* failed_map;
866 : unsigned l;
867 : char esc_buffer[ESC_MAX];
868 : char esc_buffer_alt[ESC_MAX];
869 :
870 118 : handle = handle_mapping(state, &diskmax);
871 :
872 : /* we need 1 * data + 2 * parity + 1 * zero */
873 118 : buffermax = diskmax + 2 * state->level + 1;
874 :
875 118 : buffer = malloc_nofail_vector_align(diskmax, buffermax, state->block_size, &buffer_alloc);
876 118 : if (!state->opt.skip_self)
877 0 : mtest_vector(buffermax, state->block_size, buffer);
878 :
879 : /* fill up the zero buffer */
880 118 : memset(buffer[buffermax - 1], 0, state->block_size);
881 118 : raid_zero(buffer[buffermax - 1]);
882 :
883 118 : failed = malloc_nofail(diskmax * sizeof(struct failed_struct));
884 118 : failed_map = malloc_nofail(diskmax * sizeof(unsigned));
885 :
886 118 : error = 0;
887 118 : unrecoverable_error = 0;
888 118 : recovered_error = 0;
889 :
890 : /* first count the number of blocks to process */
891 118 : countmax = 0;
892 935597 : for (i = blockstart; i < blockmax; ++i) {
893 935479 : if (!block_is_enabled(state, i, handle, diskmax))
894 25246 : continue;
895 910233 : ++countmax;
896 : }
897 :
898 : /* check all the blocks in files */
899 118 : countsize = 0;
900 118 : countpos = 0;
901 118 : state_progress_begin(state, blockstart, blockmax, countmax);
902 935597 : for (i = blockstart; i < blockmax; ++i) {
903 : unsigned failed_count;
904 : int valid_parity;
905 : snapraid_info info;
906 : int rehash;
907 :
908 935479 : if (!block_is_enabled(state, i, handle, diskmax)) {
909 : /* post process the files */
910 25246 : ret = file_post(state, fix, i, handle, diskmax);
911 25246 : if (ret == -1) {
912 : /* LCOV_EXCL_START */
913 : log_fatal("Stopping at block %u\n", i);
914 : ++unrecoverable_error;
915 : goto bail;
916 : /* LCOV_EXCL_STOP */
917 : }
918 :
919 : /* and now continue with the next block */
920 25246 : continue;
921 : }
922 :
923 : /* If we have valid parity, and it makes sense to check its content. */
924 : /* If we already know that the parity is invalid, we just read the file */
925 : /* but we don't report parity errors */
926 : /* Note that with auditonly, we anyway skip the full parity check, */
927 : /* because we also don't read it at all */
928 910233 : valid_parity = 1;
929 :
930 : /* keep track of the number of failed blocks */
931 910233 : failed_count = 0;
932 :
933 : /* get block specific info */
934 910233 : info = info_get(&state->infoarr, i);
935 :
936 : /* if we have to use the old hash */
937 910233 : rehash = info_get_rehash(info);
938 :
939 : /* for each disk, process the block */
940 6371631 : for (j = 0; j < diskmax; ++j) {
941 : int read_size;
942 : unsigned char hash[HASH_MAX];
943 : struct snapraid_disk* disk;
944 : struct snapraid_block* block;
945 : struct snapraid_file* file;
946 : block_off_t file_pos;
947 : unsigned block_state;
948 :
949 : /* if the disk position is not used */
950 5461398 : disk = handle[j].disk;
951 5461398 : if (!disk) {
952 : /* use an empty block */
953 14061 : memset(buffer[j], 0, state->block_size);
954 954823 : continue;
955 : }
956 :
957 : /* if the disk block is not used */
958 5447337 : block = fs_par2block_find(disk, i);
959 5447337 : if (block == BLOCK_NULL) {
960 : /* use an empty block */
961 192063 : memset(buffer[j], 0, state->block_size);
962 192063 : continue;
963 : }
964 :
965 : /* get the state of the block */
966 5255274 : block_state = block_state_get(block);
967 :
968 : /* if the parity is not valid */
969 5255274 : if (block_has_invalid_parity(block)) {
970 : /* mark the parity as invalid, and don't try to check/fix it */
971 : /* because it will be recomputed at the next sync */
972 94587 : valid_parity = 0;
973 : /* follow */
974 : }
975 :
976 : /* if the block is DELETED */
977 5255274 : if (block_state == BLOCK_STATE_DELETED) {
978 : /* use an empty block */
979 18454 : memset(buffer[j], 0, state->block_size);
980 :
981 : /* store it in the failed set, because potentially */
982 : /* the parity may be still computed with the previous content */
983 18454 : failed[failed_count].is_bad = 0; /* note that is_bad==0 <=> file==0 */
984 18454 : failed[failed_count].is_outofdate = 0;
985 18454 : failed[failed_count].index = j;
986 18454 : failed[failed_count].block = block;
987 18454 : failed[failed_count].disk = disk;
988 18454 : failed[failed_count].file = 0;
989 18454 : failed[failed_count].file_pos = 0;
990 18454 : failed[failed_count].handle = 0;
991 18454 : ++failed_count;
992 18454 : continue;
993 : }
994 :
995 : /* get the file of this block */
996 5236820 : file = fs_par2file_get(disk, i, &file_pos);
997 :
998 : /* if we are only hashing, we can skip excluded files and don't even read them */
999 5236820 : if (state->opt.auditonly && file_flag_has(file, FILE_IS_EXCLUDED)) {
1000 : /* use an empty block */
1001 : /* in true, this is unnecessary, because we are not checking any parity */
1002 : /* but we keep it for completeness */
1003 0 : memset(buffer[j], 0, state->block_size);
1004 0 : continue;
1005 : }
1006 :
1007 : /* if the file is closed or different than the current one */
1008 5236820 : if (handle[j].file == 0 || handle[j].file != file) {
1009 : /* close the old one, if any */
1010 2259576 : ret = handle_close(&handle[j]);
1011 2259576 : if (ret == -1) {
1012 : /* LCOV_EXCL_START */
1013 : log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc_tag(handle[j].file->sub, esc_buffer), strerror(errno));
1014 : log_fatal("DANGER! Unexpected close error in a data disk.\n");
1015 : log_fatal("Stopping at block %u\n", i);
1016 : ++unrecoverable_error;
1017 : goto bail;
1018 : /* LCOV_EXCL_STOP */
1019 : }
1020 :
1021 : /* if fixing, and the file is not excluded, we must open for writing */
1022 2259576 : if (fix && !file_flag_has(file, FILE_IS_EXCLUDED)) {
1023 : /* if fixing, create the file, open for writing and resize if required */
1024 771283 : ret = handle_create(&handle[j], file, state->file_mode);
1025 771283 : if (ret == -1) {
1026 : /* LCOV_EXCL_START */
1027 : if (errno == EACCES) {
1028 : log_fatal("WARNING! Please give write permission to the file.\n");
1029 : } else {
1030 : log_fatal("DANGER! Without a working data disk, it isn't possible to fix errors on it.\n");
1031 : }
1032 : log_fatal("Stopping at block %u\n", i);
1033 : ++unrecoverable_error;
1034 : goto bail;
1035 : /* LCOV_EXCL_STOP */
1036 : }
1037 :
1038 : /* check if the file was just created */
1039 1542566 : if (handle[j].created != 0) {
1040 : /* if fragmented, it may be reopened, so remember that the file */
1041 : /* was originally missing */
1042 113044 : file_flag_set(file, FILE_IS_CREATED);
1043 : }
1044 : } else {
1045 : /* open the file only for reading */
1046 1488293 : if (!file_flag_has(file, FILE_IS_MISSING))
1047 1372856 : ret = handle_open(&handle[j], file, state->file_mode,
1048 1372856 : log_error, state->opt.expected_missing ? log_expected : 0);
1049 : else
1050 115437 : ret = -1; /* if the file is missing, we cannot open it */
1051 1488293 : if (ret == -1) {
1052 : /* save the failed block for the check/fix */
1053 194019 : failed[failed_count].is_bad = 1;
1054 194019 : failed[failed_count].is_outofdate = 0;
1055 194019 : failed[failed_count].index = j;
1056 194019 : failed[failed_count].block = block;
1057 194019 : failed[failed_count].disk = disk;
1058 194019 : failed[failed_count].file = file;
1059 194019 : failed[failed_count].file_pos = file_pos;
1060 194019 : failed[failed_count].handle = &handle[j];
1061 194019 : ++failed_count;
1062 :
1063 194019 : log_tag("error:%u:%s:%s: Open error at position %u\n", i, disk->name, esc_tag(file->sub, esc_buffer), file_pos);
1064 194019 : ++error;
1065 :
1066 : /* mark the file as missing, to avoid to retry to open it again */
1067 : /* note that this can be done only if we are not fixing it */
1068 : /* otherwise, it could be recreated */
1069 194019 : file_flag_set(file, FILE_IS_MISSING);
1070 194019 : continue;
1071 : }
1072 : }
1073 :
1074 : /* if it's the first open, and not excluded */
1075 2065557 : if (!file_flag_has(file, FILE_IS_OPENED)
1076 2050191 : && !file_flag_has(file, FILE_IS_EXCLUDED)) {
1077 :
1078 : /* check if the file is changed */
1079 1998760 : if (handle[j].st.st_size != file->size
1080 1805894 : || handle[j].st.st_mtime != file->mtime_sec
1081 1781363 : || STAT_NSEC(&handle[j].st) != file->mtime_nsec
1082 : /* don't check the inode to support file-system without persistent inodes */
1083 : ) {
1084 : /* report that the file is not synced */
1085 217397 : file_flag_set(file, FILE_IS_UNSYNCED);
1086 : }
1087 : }
1088 :
1089 : /* if it's the first open, and not excluded and larger */
1090 2065557 : if (!file_flag_has(file, FILE_IS_OPENED)
1091 2050191 : && !file_flag_has(file, FILE_IS_EXCLUDED)
1092 1998760 : && !(state->opt.syncedonly && file_flag_has(file, FILE_IS_UNSYNCED))
1093 1998760 : && handle[j].st.st_size > file->size
1094 : ) {
1095 8819 : log_error("File '%s' is larger than expected.\n", handle[j].path);
1096 8819 : log_tag("error:%u:%s:%s: Size error\n", i, disk->name, esc_tag(file->sub, esc_buffer));
1097 8819 : ++error;
1098 :
1099 8819 : if (fix) {
1100 3254 : ret = handle_truncate(&handle[j], file);
1101 3254 : if (ret == -1) {
1102 : /* LCOV_EXCL_START */
1103 : log_fatal("DANGER! Unexpected truncate error in a data disk, it isn't possible to fix.\n");
1104 : log_fatal("Stopping at block %u\n", i);
1105 : ++unrecoverable_error;
1106 : goto bail;
1107 : /* LCOV_EXCL_STOP */
1108 : }
1109 :
1110 3254 : log_tag("fixed:%u:%s:%s: Fixed size\n", i, disk->name, esc_tag(file->sub, esc_buffer));
1111 3254 : ++recovered_error;
1112 : }
1113 : }
1114 :
1115 : /* mark the file as opened at least one time */
1116 : /* this is used to avoid to check the unsynced and size */
1117 : /* more than one time, in case the file is reopened later */
1118 2065557 : file_flag_set(file, FILE_IS_OPENED);
1119 : }
1120 :
1121 : /* read from the file */
1122 5042801 : read_size = handle_read(&handle[j], file_pos, buffer[j], state->block_size,
1123 5042801 : log_error, state->opt.expected_missing ? log_expected : 0);
1124 5042801 : if (read_size == -1) {
1125 : /* save the failed block for the check/fix */
1126 442126 : failed[failed_count].is_bad = 1; /* it's bad because we cannot read it */
1127 442126 : failed[failed_count].is_outofdate = 0;
1128 442126 : failed[failed_count].index = j;
1129 442126 : failed[failed_count].block = block;
1130 442126 : failed[failed_count].disk = disk;
1131 442126 : failed[failed_count].file = file;
1132 442126 : failed[failed_count].file_pos = file_pos;
1133 442126 : failed[failed_count].handle = &handle[j];
1134 442126 : ++failed_count;
1135 :
1136 442126 : log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc_tag(file->sub, esc_buffer), file_pos);
1137 442126 : ++error;
1138 442126 : continue;
1139 : }
1140 :
1141 4600675 : countsize += read_size;
1142 :
1143 : /* always insert CHG blocks, the repair functions needs all of them */
1144 : /* because the parity may be still referring at the old state */
1145 : /* and the repair must be aware of it */
1146 4600675 : if (block_state == BLOCK_STATE_CHG) {
1147 : /* we DO NOT mark them as bad to avoid to overwrite them with wrong data. */
1148 : /* if we don't have a hash, we always assume the first read of the block correct. */
1149 13843 : failed[failed_count].is_bad = 0; /* we assume the CHG block correct */
1150 13843 : failed[failed_count].is_outofdate = 0;
1151 13843 : failed[failed_count].index = j;
1152 13843 : failed[failed_count].block = block;
1153 13843 : failed[failed_count].disk = disk;
1154 13843 : failed[failed_count].file = file;
1155 13843 : failed[failed_count].file_pos = file_pos;
1156 13843 : failed[failed_count].handle = &handle[j];
1157 13843 : ++failed_count;
1158 13843 : continue;
1159 : }
1160 :
1161 4586832 : assert(block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP);
1162 :
1163 : /* compute the hash of the block just read */
1164 4586832 : if (rehash) {
1165 27243 : memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
1166 : } else {
1167 4559589 : memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
1168 : }
1169 :
1170 : /* compare the hash */
1171 4586832 : if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) {
1172 12768 : unsigned diff = memdiff(hash, block->hash, BLOCK_HASH_SIZE);
1173 :
1174 : /* save the failed block for the check/fix */
1175 12768 : failed[failed_count].is_bad = 1; /* it's bad because the hash doesn't match */
1176 12768 : failed[failed_count].is_outofdate = 0;
1177 12768 : failed[failed_count].index = j;
1178 12768 : failed[failed_count].block = block;
1179 12768 : failed[failed_count].disk = disk;
1180 12768 : failed[failed_count].file = file;
1181 12768 : failed[failed_count].file_pos = file_pos;
1182 12768 : failed[failed_count].handle = &handle[j];
1183 12768 : ++failed_count;
1184 :
1185 12768 : log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc_tag(file->sub, esc_buffer), file_pos, diff);
1186 12768 : ++error;
1187 12768 : continue;
1188 : }
1189 :
1190 : /* always insert REP blocks, the repair functions needs all of them */
1191 : /* because the parity may be still referring at the old state */
1192 : /* and the repair must be aware of it */
1193 4574064 : if (block_state == BLOCK_STATE_REP) {
1194 53428 : failed[failed_count].is_bad = 0; /* it's not bad */
1195 53428 : failed[failed_count].is_outofdate = 0;
1196 53428 : failed[failed_count].index = j;
1197 53428 : failed[failed_count].block = block;
1198 53428 : failed[failed_count].disk = disk;
1199 53428 : failed[failed_count].file = file;
1200 53428 : failed[failed_count].file_pos = file_pos;
1201 53428 : failed[failed_count].handle = &handle[j];
1202 53428 : ++failed_count;
1203 53428 : continue;
1204 : }
1205 : }
1206 :
1207 : /* now read and check the parity if requested */
1208 910233 : if (!state->opt.auditonly) {
1209 : void* buffer_recov[LEV_MAX];
1210 : void* buffer_zero;
1211 :
1212 : /* buffers for parity read and not computed */
1213 4170787 : for (l = 0; l < state->level; ++l)
1214 3299829 : buffer_recov[l] = buffer[diskmax + state->level + l];
1215 2796877 : for (; l < LEV_MAX; ++l)
1216 1925919 : buffer_recov[l] = 0;
1217 :
1218 : /* the zero buffer is the last one */
1219 870958 : buffer_zero = buffer[buffermax - 1];
1220 :
1221 : /* read the parity */
1222 4170787 : for (l = 0; l < state->level; ++l) {
1223 3299829 : if (parity[l]) {
1224 3285927 : ret = parity_read(parity[l], i, buffer_recov[l], state->block_size, log_error);
1225 3285927 : if (ret == -1) {
1226 11360 : buffer_recov[l] = 0; /* no parity to use */
1227 :
1228 11360 : log_tag("parity_error:%u:%s: Read error\n", i, lev_config_name(l));
1229 11360 : ++error;
1230 : }
1231 : } else {
1232 13902 : buffer_recov[l] = 0;
1233 : }
1234 : }
1235 :
1236 : /* try all the recovering strategies */
1237 870958 : ret = repair(state, rehash, i, diskmax, failed, failed_map, failed_count, buffer, buffer_recov, buffer_zero);
1238 870958 : if (ret != 0) {
1239 : /* increment the number of errors */
1240 47935 : if (ret > 0)
1241 4811 : error += ret;
1242 47935 : ++unrecoverable_error;
1243 :
1244 : /* print a list of all the errors in files */
1245 219126 : for (j = 0; j < failed_count; ++j) {
1246 171191 : if (failed[j].is_bad)
1247 166669 : log_tag("unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].disk->name, esc_tag(failed[j].file->sub, esc_buffer), failed[j].file_pos);
1248 : }
1249 :
1250 : /* keep track of damaged files */
1251 219126 : for (j = 0; j < failed_count; ++j) {
1252 171191 : if (failed[j].is_bad)
1253 166669 : file_flag_set(failed[j].file, FILE_IS_DAMAGED);
1254 : }
1255 : } else {
1256 : /* now counts partial recovers */
1257 : /* note that this could happen only when we have an incomplete 'sync' */
1258 : /* and that we have recovered is the state before the 'sync' */
1259 823023 : int partial_recover_error = 0;
1260 :
1261 : /* print a list of all the errors in files */
1262 1380653 : for (j = 0; j < failed_count; ++j) {
1263 557630 : if (failed[j].is_bad && failed[j].is_outofdate) {
1264 8 : ++partial_recover_error;
1265 8 : log_tag("unrecoverable:%u:%s:%s: Unrecoverable unsynced error at position %u\n", i, failed[j].disk->name, esc_tag(failed[j].file->sub, esc_buffer), failed[j].file_pos);
1266 : }
1267 : }
1268 823023 : if (partial_recover_error != 0) {
1269 8 : error += partial_recover_error;
1270 8 : ++unrecoverable_error;
1271 : }
1272 :
1273 : /* now check parities, but only if all the blocks have it computed */
1274 : /* if you check/fix after a partial sync, it's OK to have parity errors on the blocks with invalid parity */
1275 : /* and doesn't make sense to try to fix it */
1276 823023 : if (valid_parity) {
1277 : /* check the parity */
1278 3728946 : for (l = 0; l < state->level; ++l) {
1279 2967508 : if (buffer_recov[l] != 0 && memcmp(buffer_recov[l], buffer[diskmax + l], state->block_size) != 0) {
1280 78680 : unsigned diff = memdiff(buffer_recov[l], buffer[diskmax + l], state->block_size);
1281 :
1282 : /* mark that the read parity is wrong, setting ptr to 0 */
1283 78680 : buffer_recov[l] = 0;
1284 :
1285 78680 : log_tag("parity_error:%u:%s: Data error, diff bits %u\n", i, lev_config_name(l), diff);
1286 78680 : ++error;
1287 : }
1288 : }
1289 : }
1290 :
1291 : /* now write recovered files */
1292 823023 : if (fix) {
1293 : /* update the fixed files */
1294 652640 : for (j = 0; j < failed_count; ++j) {
1295 : /* nothing to do if it doesn't need recovering */
1296 337975 : if (!failed[j].is_bad)
1297 40318 : continue;
1298 :
1299 : /* do not fix if the file is excluded */
1300 297657 : if (file_flag_has(failed[j].file, FILE_IS_EXCLUDED)
1301 288377 : || (state->opt.syncedonly && file_flag_has(failed[j].file, FILE_IS_UNSYNCED)))
1302 9280 : continue;
1303 :
1304 288377 : ret = handle_write(failed[j].handle, failed[j].file_pos, buffer[failed[j].index], state->block_size);
1305 288377 : if (ret == -1) {
1306 : /* LCOV_EXCL_START */
1307 : /* mark the file as damaged */
1308 : file_flag_set(failed[j].file, FILE_IS_DAMAGED);
1309 :
1310 : if (errno == EACCES) {
1311 : log_fatal("WARNING! Please give write permission to the file.\n");
1312 : } else {
1313 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1314 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1315 : }
1316 : log_fatal("Stopping at block %u\n", i);
1317 : ++unrecoverable_error;
1318 : goto bail;
1319 : /* LCOV_EXCL_STOP */
1320 : }
1321 :
1322 : /* if we are not sure that the recovered content is uptodate */
1323 288377 : if (failed[j].is_outofdate) {
1324 : /* mark the file as damaged */
1325 8 : file_flag_set(failed[j].file, FILE_IS_DAMAGED);
1326 8 : continue;
1327 : }
1328 :
1329 : /* mark the file as containing some fixes */
1330 : /* note that it could be also marked as damaged in other iterations */
1331 288369 : file_flag_set(failed[j].file, FILE_IS_FIXED);
1332 :
1333 288369 : log_tag("fixed:%u:%s:%s: Fixed data error at position %u\n", i, failed[j].disk->name, esc_tag(failed[j].file->sub, esc_buffer), failed[j].file_pos);
1334 288369 : ++recovered_error;
1335 : }
1336 :
1337 : /* update parity only if all the blocks have it computed */
1338 : /* if you check/fix after a partial sync, you do not want to fix parity */
1339 : /* for blocks that are going to have it computed in the sync completion */
1340 314665 : if (valid_parity) {
1341 : /* update the parity */
1342 964055 : for (l = 0; l < state->level; ++l) {
1343 : /* if the parity on disk is wrong */
1344 698570 : if (buffer_recov[l] == 0
1345 : /* and we have access at the parity */
1346 72150 : && parity[l] != 0
1347 : /* and the parity is not excluded */
1348 67463 : && !state->parity[l].is_excluded
1349 : ) {
1350 67463 : ret = parity_write(parity[l], i, buffer[diskmax + l], state->block_size);
1351 67463 : if (ret == -1) {
1352 : /* LCOV_EXCL_START */
1353 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1354 : log_fatal("WARNING! Without a working %s disk, it isn't possible to fix errors on it.\n", lev_name(l));
1355 : log_fatal("Stopping at block %u\n", i);
1356 : ++unrecoverable_error;
1357 : goto bail;
1358 : /* LCOV_EXCL_STOP */
1359 : }
1360 :
1361 67463 : log_tag("parity_fixed:%u:%s: Fixed data error\n", i, lev_config_name(l));
1362 67463 : ++recovered_error;
1363 : }
1364 : }
1365 : }
1366 : } else {
1367 : /* if we are not fixing, we just set the FIXED flag */
1368 : /* meaning that we could fix this file if we try */
1369 728013 : for (j = 0; j < failed_count; ++j) {
1370 219655 : if (failed[j].is_bad) {
1371 178770 : file_flag_set(failed[j].file, FILE_IS_FIXED);
1372 : }
1373 : }
1374 : }
1375 : }
1376 : } else {
1377 : /* if we are not checking, we just set the DAMAGED flag */
1378 : /* to report that the file is damaged, and we don't know if we can fix it */
1379 45092 : for (j = 0; j < failed_count; ++j) {
1380 5817 : if (failed[j].is_bad) {
1381 5817 : file_flag_set(failed[j].file, FILE_IS_DAMAGED);
1382 : }
1383 : }
1384 : }
1385 :
1386 : /* post process the files */
1387 910233 : ret = file_post(state, fix, i, handle, diskmax);
1388 910233 : if (ret == -1) {
1389 : /* LCOV_EXCL_START */
1390 : log_fatal("Stopping at block %u\n", i);
1391 : ++unrecoverable_error;
1392 : goto bail;
1393 : /* LCOV_EXCL_STOP */
1394 : }
1395 :
1396 : /* count the number of processed block */
1397 910233 : ++countpos;
1398 :
1399 : /* progress */
1400 910233 : if (state_progress(state, 0, i, countpos, countmax, countsize)) {
1401 : /* LCOV_EXCL_START */
1402 : break;
1403 : /* LCOV_EXCL_STOP */
1404 : }
1405 : }
1406 :
1407 : /* for each disk, recover empty files, symlinks and empty dirs */
1408 826 : for (i = 0; i < diskmax; ++i) {
1409 : tommy_node* node;
1410 : struct snapraid_disk* disk;
1411 :
1412 708 : if (!handle[i].disk)
1413 3 : continue;
1414 :
1415 : /* for each empty file in the disk */
1416 705 : disk = handle[i].disk;
1417 705 : node = disk->filelist;
1418 2189135 : while (node) {
1419 : char path[PATH_MAX];
1420 : struct stat st;
1421 : struct snapraid_file* file;
1422 2187725 : int unsuccesful = 0;
1423 :
1424 2187725 : file = node->data;
1425 2187725 : node = node->next; /* next node */
1426 :
1427 : /* if not empty, it's already checked and continue to the next one */
1428 2187725 : if (file->size != 0) {
1429 4369715 : continue;
1430 : }
1431 :
1432 : /* if excluded continue to the next one */
1433 2952 : if (file_flag_has(file, FILE_IS_EXCLUDED)) {
1434 169 : continue;
1435 : }
1436 :
1437 : /* stat the file */
1438 2783 : pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
1439 2783 : ret = stat(path, &st);
1440 2783 : if (ret == -1) {
1441 179 : unsuccesful = 1;
1442 :
1443 179 : log_error("Error stating empty file '%s'. %s.\n", path, strerror(errno));
1444 179 : log_tag("error:%s:%s: Empty file stat error\n", disk->name, esc_tag(file->sub, esc_buffer));
1445 179 : ++error;
1446 2604 : } else if (!S_ISREG(st.st_mode)) {
1447 0 : unsuccesful = 1;
1448 :
1449 0 : log_tag("error:%s:%s: Empty file error for not regular file\n", disk->name, esc_tag(file->sub, esc_buffer));
1450 0 : ++error;
1451 2604 : } else if (st.st_size != 0) {
1452 6 : unsuccesful = 1;
1453 :
1454 6 : log_tag("error:%s:%s: Empty file error for size '%" PRIu64 "'\n", disk->name, esc_tag(file->sub, esc_buffer), st.st_size);
1455 6 : ++error;
1456 : }
1457 :
1458 2783 : if (fix && unsuccesful) {
1459 : int f;
1460 :
1461 : /* create the ancestor directories */
1462 159 : ret = mkancestor(path);
1463 159 : if (ret != 0) {
1464 : /* LCOV_EXCL_START */
1465 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1466 : log_fatal("Stopping\n");
1467 : ++unrecoverable_error;
1468 : goto bail;
1469 : /* LCOV_EXCL_STOP */
1470 : }
1471 :
1472 : /* create it */
1473 : /* O_NOFOLLOW: do not follow links to ensure to open the real file */
1474 159 : f = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_NOFOLLOW, 0600);
1475 159 : if (f == -1) {
1476 : /* LCOV_EXCL_START */
1477 : log_fatal("Error creating empty file '%s'. %s.\n", path, strerror(errno));
1478 : if (errno == EACCES) {
1479 : log_fatal("WARNING! Please give write permission to the file.\n");
1480 : } else {
1481 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1482 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1483 : }
1484 : log_fatal("Stopping\n");
1485 : ++unrecoverable_error;
1486 : goto bail;
1487 : /* LCOV_EXCL_STOP */
1488 : }
1489 :
1490 : /* set the original modification time */
1491 159 : ret = fmtime(f, file->mtime_sec, file->mtime_nsec);
1492 159 : if (ret != 0) {
1493 : /* LCOV_EXCL_START */
1494 : close(f);
1495 :
1496 : log_fatal("Error timing file '%s'. %s.\n", file->sub, strerror(errno));
1497 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1498 : log_fatal("Stopping\n");
1499 : ++unrecoverable_error;
1500 : goto bail;
1501 : /* LCOV_EXCL_STOP */
1502 : }
1503 :
1504 : /* close it */
1505 159 : ret = close(f);
1506 159 : if (ret != 0) {
1507 : /* LCOV_EXCL_START */
1508 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1509 : log_fatal("Stopping\n");
1510 : ++unrecoverable_error;
1511 : goto bail;
1512 : /* LCOV_EXCL_STOP */
1513 : }
1514 :
1515 159 : log_tag("fixed:%s:%s: Fixed empty file\n", disk->name, esc_tag(file->sub, esc_buffer));
1516 159 : ++recovered_error;
1517 :
1518 159 : log_tag("status:recovered:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
1519 159 : msg_info("recovered %s\n", fmt_term(disk, file->sub, esc_buffer));
1520 : }
1521 : }
1522 :
1523 : /* for each link in the disk */
1524 705 : disk = handle[i].disk;
1525 705 : node = disk->linklist;
1526 71537 : while (node) {
1527 : char path[PATH_MAX];
1528 : char pathto[PATH_MAX];
1529 : char linkto[PATH_MAX];
1530 : struct stat st;
1531 : struct stat stto;
1532 : struct snapraid_link* slink;
1533 70127 : int unsuccesful = 0;
1534 70127 : int unrecoverable = 0;
1535 :
1536 70127 : slink = node->data;
1537 70127 : node = node->next; /* next node */
1538 :
1539 : /* if excluded continue to the next one */
1540 70127 : if (link_flag_has(slink, FILE_IS_EXCLUDED)) {
1541 3277 : continue;
1542 : }
1543 :
1544 66850 : if (link_flag_has(slink, FILE_IS_HARDLINK)) {
1545 : /* stat the link */
1546 318 : pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub);
1547 318 : ret = stat(path, &st);
1548 318 : if (ret == -1) {
1549 36 : unsuccesful = 1;
1550 :
1551 36 : log_error("Error stating hardlink '%s'. %s.\n", path, strerror(errno));
1552 36 : log_tag("hardlink_error:%s:%s:%s: Hardlink stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
1553 36 : ++error;
1554 282 : } else if (!S_ISREG(st.st_mode)) {
1555 0 : unsuccesful = 1;
1556 :
1557 0 : log_tag("hardlink_error:%s:%s:%s: Hardlink error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
1558 0 : ++error;
1559 : }
1560 :
1561 : /* stat the "to" file */
1562 318 : pathprint(pathto, sizeof(pathto), "%s%s", disk->dir, slink->linkto);
1563 318 : ret = stat(pathto, &stto);
1564 318 : if (ret == -1) {
1565 24 : unsuccesful = 1;
1566 :
1567 24 : if (errno == ENOENT) {
1568 24 : unrecoverable = 1;
1569 24 : if (fix) {
1570 : /* if the target doesn't exist, it's unrecoverable */
1571 : /* because we cannot create an hardlink of a file that */
1572 : /* doesn't exists */
1573 12 : ++unrecoverable_error;
1574 : } else {
1575 : /* but in check, we can assume that fixing will recover */
1576 : /* such missing file, so we assume a less drastic error */
1577 12 : ++error;
1578 : }
1579 : }
1580 :
1581 24 : log_error("Error stating hardlink-to '%s'. %s.\n", pathto, strerror(errno));
1582 24 : log_tag("hardlink_error:%s:%s:%s: Hardlink to stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
1583 24 : ++error;
1584 294 : } else if (!S_ISREG(stto.st_mode)) {
1585 0 : unsuccesful = 1;
1586 :
1587 0 : log_tag("hardlink_error:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
1588 0 : ++error;
1589 294 : } else if (!unsuccesful && st.st_ino != stto.st_ino) {
1590 0 : unsuccesful = 1;
1591 :
1592 0 : log_error("Mismatch hardlink '%s' and '%s'. Different inode.\n", path, pathto);
1593 0 : log_tag("hardlink_error:%s:%s:%s: Hardlink mismatch for different inode\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
1594 0 : ++error;
1595 : }
1596 : } else {
1597 : /* read the symlink */
1598 66532 : pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub);
1599 66532 : ret = readlink(path, linkto, sizeof(linkto));
1600 66532 : if (ret < 0) {
1601 4278 : unsuccesful = 1;
1602 :
1603 4278 : log_error("Error reading symlink '%s'. %s.\n", path, strerror(errno));
1604 4278 : log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer));
1605 4278 : ++error;
1606 62254 : } else if (ret >= PATH_MAX) {
1607 0 : unsuccesful = 1;
1608 :
1609 0 : log_error("Error reading symlink '%s'. Symlink too long.\n", path);
1610 0 : log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer));
1611 0 : ++error;
1612 : } else {
1613 62254 : linkto[ret] = 0;
1614 :
1615 62254 : if (strcmp(linkto, slink->linkto) != 0) {
1616 555 : unsuccesful = 1;
1617 :
1618 555 : log_tag("symlink_error:%s:%s: Symlink data error '%s' instead of '%s'\n", disk->name, esc_tag(slink->sub, esc_buffer), linkto, slink->linkto);
1619 555 : ++error;
1620 : }
1621 : }
1622 : }
1623 :
1624 66850 : if (fix && unsuccesful && !unrecoverable) {
1625 : /* create the ancestor directories */
1626 3941 : ret = mkancestor(path);
1627 3941 : if (ret != 0) {
1628 : /* LCOV_EXCL_START */
1629 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1630 : log_fatal("Stopping\n");
1631 : ++unrecoverable_error;
1632 : goto bail;
1633 : /* LCOV_EXCL_STOP */
1634 : }
1635 :
1636 : /* if it exists, it must be deleted before recreating */
1637 3941 : ret = remove(path);
1638 3941 : if (ret != 0 && errno != ENOENT) {
1639 : /* LCOV_EXCL_START */
1640 : log_fatal("Error removing '%s'. %s.\n", path, strerror(errno));
1641 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1642 : log_fatal("Stopping\n");
1643 : ++unrecoverable_error;
1644 : goto bail;
1645 : /* LCOV_EXCL_STOP */
1646 : }
1647 :
1648 : /* create it */
1649 3941 : if (link_flag_has(slink, FILE_IS_HARDLINK)) {
1650 12 : ret = hardlink(pathto, path);
1651 12 : if (ret != 0) {
1652 : /* LCOV_EXCL_START */
1653 : log_fatal("Error writing hardlink '%s' to '%s'. %s.\n", path, pathto, strerror(errno));
1654 : if (errno == EACCES) {
1655 : log_fatal("WARNING! Please give write permission to the hardlink.\n");
1656 : } else {
1657 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1658 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1659 : }
1660 : log_fatal("Stopping\n");
1661 : ++unrecoverable_error;
1662 : goto bail;
1663 : /* LCOV_EXCL_STOP */
1664 : }
1665 :
1666 12 : log_tag("hardlink_fixed:%s:%s: Fixed hardlink error\n", disk->name, esc_tag(slink->sub, esc_buffer));
1667 12 : ++recovered_error;
1668 : } else {
1669 3929 : ret = symlink(slink->linkto, path);
1670 3929 : if (ret != 0) {
1671 : /* LCOV_EXCL_START */
1672 : log_fatal("Error writing symlink '%s' to '%s'. %s.\n", path, slink->linkto, strerror(errno));
1673 : if (errno == EACCES) {
1674 : log_fatal("WARNING! Please give write permission to the symlink.\n");
1675 : } else {
1676 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1677 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1678 : }
1679 : log_fatal("Stopping\n");
1680 : ++unrecoverable_error;
1681 : goto bail;
1682 : /* LCOV_EXCL_STOP */
1683 : }
1684 :
1685 3929 : log_tag("symlink_fixed:%s:%s: Fixed symlink error\n", disk->name, esc_tag(slink->sub, esc_buffer));
1686 3929 : ++recovered_error;
1687 : }
1688 :
1689 3941 : log_tag("status:recovered:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
1690 3941 : msg_info("recovered %s\n", fmt_term(disk, slink->sub, esc_buffer));
1691 : }
1692 : }
1693 :
1694 : /* for each dir in the disk */
1695 705 : disk = handle[i].disk;
1696 705 : node = disk->dirlist;
1697 1850 : while (node) {
1698 : char path[PATH_MAX];
1699 : struct stat st;
1700 : struct snapraid_dir* dir;
1701 440 : int unsuccesful = 0;
1702 :
1703 440 : dir = node->data;
1704 440 : node = node->next; /* next node */
1705 :
1706 : /* if excluded continue to the next one */
1707 440 : if (dir_flag_has(dir, FILE_IS_EXCLUDED)) {
1708 19 : continue;
1709 : }
1710 :
1711 : /* stat the dir */
1712 421 : pathprint(path, sizeof(path), "%s%s", disk->dir, dir->sub);
1713 421 : ret = stat(path, &st);
1714 421 : if (ret == -1) {
1715 23 : unsuccesful = 1;
1716 :
1717 23 : log_error("Error stating dir '%s'. %s.\n", path, strerror(errno));
1718 23 : log_tag("dir_error:%s:%s: Dir stat error\n", disk->name, esc_tag(dir->sub, esc_buffer));
1719 23 : ++error;
1720 398 : } else if (!S_ISDIR(st.st_mode)) {
1721 0 : unsuccesful = 1;
1722 :
1723 0 : log_tag("dir_error:%s:%s: Dir error for not directory\n", disk->name, esc_tag(dir->sub, esc_buffer));
1724 0 : ++error;
1725 : }
1726 :
1727 421 : if (fix && unsuccesful) {
1728 : /* create the ancestor directories */
1729 21 : ret = mkancestor(path);
1730 21 : if (ret != 0) {
1731 : /* LCOV_EXCL_START */
1732 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1733 : log_fatal("Stopping\n");
1734 : ++unrecoverable_error;
1735 : goto bail;
1736 : /* LCOV_EXCL_STOP */
1737 : }
1738 :
1739 : /* create it */
1740 21 : ret = mkdir(path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
1741 21 : if (ret != 0) {
1742 : /* LCOV_EXCL_START */
1743 : log_fatal("Error creating dir '%s'. %s.\n", path, strerror(errno));
1744 : if (errno == EACCES) {
1745 : log_fatal("WARNING! Please give write permission to the dir.\n");
1746 : } else {
1747 : /* we do not use DANGER because it could be ENOSPC which is not always correctly reported */
1748 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1749 : }
1750 : log_fatal("Stopping\n");
1751 : ++unrecoverable_error;
1752 : goto bail;
1753 : /* LCOV_EXCL_STOP */
1754 : }
1755 :
1756 21 : log_tag("dir_fixed:%s:%s: Fixed dir error\n", disk->name, esc_tag(dir->sub, esc_buffer));
1757 21 : ++recovered_error;
1758 :
1759 21 : log_tag("status:recovered:%s:%s\n", disk->name, esc_tag(dir->sub, esc_buffer));
1760 21 : msg_info("recovered %s\n", fmt_term(disk, dir->sub, esc_buffer));
1761 : }
1762 : }
1763 : }
1764 :
1765 118 : state_progress_end(state, countpos, countmax, countsize);
1766 :
1767 : bail:
1768 : /* close all the files left open */
1769 826 : for (j = 0; j < diskmax; ++j) {
1770 708 : struct snapraid_file* file = handle[j].file;
1771 708 : struct snapraid_disk* disk = handle[j].disk;
1772 708 : ret = handle_close(&handle[j]);
1773 708 : if (ret == -1) {
1774 : /* LCOV_EXCL_START */
1775 : log_tag("error:%u:%s:%s: Close error. %s\n", blockmax, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
1776 : log_fatal("DANGER! Unexpected close error in a data disk.\n");
1777 : ++unrecoverable_error;
1778 : /* continue, as we are already exiting */
1779 : /* LCOV_EXCL_STOP */
1780 : }
1781 : }
1782 :
1783 : /* remove all the files created from scratch that have not finished the processing */
1784 : /* it happens only when aborting pressing Ctrl+C or other reason. */
1785 118 : if (fix) {
1786 : /* for each disk */
1787 315 : for (i = 0; i < diskmax; ++i) {
1788 : tommy_node* node;
1789 : struct snapraid_disk* disk;
1790 :
1791 270 : if (!handle[i].disk)
1792 1 : continue;
1793 :
1794 : /* for each file in the disk */
1795 269 : disk = handle[i].disk;
1796 269 : node = disk->filelist;
1797 870849 : while (node) {
1798 : char path[PATH_MAX];
1799 : struct snapraid_file* file;
1800 :
1801 870311 : file = node->data;
1802 870311 : node = node->next; /* next node */
1803 :
1804 : /* if the file was not created, meaning that it was already existing */
1805 870311 : if (!file_flag_has(file, FILE_IS_CREATED)) {
1806 : /* nothing to do */
1807 1627578 : continue;
1808 : }
1809 :
1810 : /* if processing was finished */
1811 113044 : if (file_flag_has(file, FILE_IS_FINISHED)) {
1812 : /* nothing to do */
1813 113044 : continue;
1814 : }
1815 :
1816 : /* if the file was originally missing, and processing not yet finished */
1817 : /* we have to throw it away to ensure that at the next run we will retry */
1818 : /* to fix it, in case we select to undelete missing files */
1819 0 : pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
1820 :
1821 0 : ret = remove(path);
1822 0 : if (ret != 0) {
1823 : /* LCOV_EXCL_START */
1824 : log_fatal("Error removing '%s'. %s.\n", path, strerror(errno));
1825 : log_fatal("WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
1826 : ++unrecoverable_error;
1827 : /* continue, as we are already exiting */
1828 : /* LCOV_EXCL_STOP */
1829 : }
1830 : }
1831 : }
1832 : }
1833 :
1834 118 : if (error || recovered_error || unrecoverable_error) {
1835 67 : msg_status("\n");
1836 67 : msg_status("%8u errors\n", error);
1837 67 : if (fix) {
1838 43 : msg_status("%8u recovered errors\n", recovered_error);
1839 : }
1840 134 : if (unrecoverable_error) {
1841 13 : msg_status("%8u UNRECOVERABLE errors\n", unrecoverable_error);
1842 : } else {
1843 : /* without checking, we don't know if they are really recoverable or not */
1844 54 : if (!state->opt.auditonly)
1845 52 : msg_status("%8u unrecoverable errors\n", unrecoverable_error);
1846 54 : if (fix)
1847 34 : msg_status("Everything OK\n");
1848 : }
1849 : } else {
1850 51 : msg_status("Everything OK\n");
1851 : }
1852 :
1853 118 : if (error && !fix)
1854 24 : log_fatal("WARNING! There are errors!\n");
1855 118 : if (unrecoverable_error)
1856 13 : log_fatal("DANGER! There are unrecoverable errors!\n");
1857 :
1858 118 : log_tag("summary:error:%u\n", error);
1859 118 : if (fix)
1860 45 : log_tag("summary:error_recovered:%u\n", recovered_error);
1861 118 : if (!state->opt.auditonly)
1862 113 : log_tag("summary:error_unrecoverable:%u\n", unrecoverable_error);
1863 118 : if (fix) {
1864 45 : if (error + recovered_error + unrecoverable_error == 0)
1865 2 : log_tag("summary:exit:ok\n");
1866 43 : else if (unrecoverable_error == 0)
1867 34 : log_tag("summary:exit:recovered\n");
1868 : else
1869 9 : log_tag("summary:exit:unrecoverable\n");
1870 73 : } else if (!state->opt.auditonly) {
1871 68 : if (error + unrecoverable_error == 0)
1872 46 : log_tag("summary:exit:ok\n");
1873 22 : else if (unrecoverable_error == 0)
1874 18 : log_tag("summary:exit:recoverable\n");
1875 : else
1876 4 : log_tag("summary:exit:unrecoverable\n");
1877 : } else { /* audit only */
1878 5 : if (error == 0)
1879 3 : log_tag("summary:exit:ok\n");
1880 : else
1881 2 : log_tag("summary:exit:error\n");
1882 : }
1883 118 : log_flush();
1884 :
1885 118 : free(failed);
1886 118 : free(failed_map);
1887 118 : free(handle);
1888 118 : free(buffer_alloc);
1889 118 : free(buffer);
1890 :
1891 : /* fail if some error are present after the run */
1892 118 : if (fix) {
1893 45 : if (state->opt.expect_unrecoverable) {
1894 9 : if (unrecoverable_error == 0)
1895 0 : return -1;
1896 : } else {
1897 36 : if (unrecoverable_error != 0)
1898 0 : return -1;
1899 : }
1900 : } else {
1901 73 : if (state->opt.expect_unrecoverable) {
1902 4 : if (unrecoverable_error == 0)
1903 0 : return -1;
1904 69 : } else if (state->opt.expect_recoverable) {
1905 20 : if (unrecoverable_error != 0 || error == 0)
1906 0 : return -1;
1907 : } else {
1908 49 : if (error != 0 || unrecoverable_error != 0)
1909 0 : return -1;
1910 : }
1911 : }
1912 :
1913 118 : return 0;
1914 : }
1915 :
1916 120 : int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount)
1917 : {
1918 : block_off_t blockmax;
1919 : data_off_t size;
1920 : int ret;
1921 : struct snapraid_parity_handle parity[LEV_MAX];
1922 : struct snapraid_parity_handle* parity_ptr[LEV_MAX];
1923 : unsigned error;
1924 : unsigned l;
1925 :
1926 120 : msg_progress("Initializing...\n");
1927 :
1928 120 : blockmax = parity_allocated_size(state);
1929 120 : size = blockmax * (data_off_t)state->block_size;
1930 :
1931 120 : if (blockstart > blockmax) {
1932 : /* LCOV_EXCL_START */
1933 : log_fatal("Error in the specified starting block %u. It's bigger than the parity size %u.\n", blockstart, blockmax);
1934 : exit(EXIT_FAILURE);
1935 : /* LCOV_EXCL_STOP */
1936 : }
1937 :
1938 : /* adjust the number of block to process */
1939 120 : if (blockcount != 0 && blockstart + blockcount < blockmax) {
1940 0 : blockmax = blockstart + blockcount;
1941 : }
1942 :
1943 120 : if (fix) {
1944 : /* if fixing, create the file and open for writing */
1945 : /* if it fails, we cannot continue */
1946 180 : for (l = 0; l < state->level; ++l) {
1947 : /* skip parity disks that are not accessible */
1948 135 : if (state->parity[l].skip_access) {
1949 1 : parity_ptr[l] = 0;
1950 1 : continue;
1951 : }
1952 :
1953 134 : parity_ptr[l] = &parity[l];
1954 134 : ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
1955 134 : if (ret == -1) {
1956 : /* LCOV_EXCL_START */
1957 : log_fatal("WARNING! Without an accessible %s file, it isn't possible to fix any error.\n", lev_name(l));
1958 : exit(EXIT_FAILURE);
1959 : /* LCOV_EXCL_STOP */
1960 : }
1961 :
1962 134 : ret = parity_chsize(parity_ptr[l], &state->parity[l], 0, size, state->block_size, state->opt.skip_fallocate, state->opt.skip_space_holder);
1963 134 : if (ret == -1) {
1964 : /* LCOV_EXCL_START */
1965 : log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
1966 : exit(EXIT_FAILURE);
1967 : /* LCOV_EXCL_STOP */
1968 : }
1969 : }
1970 75 : } else if (!state->opt.auditonly) {
1971 : /* if checking, open the file for reading */
1972 : /* it may fail if the file doesn't exist, in this case we continue to check the files */
1973 393 : for (l = 0; l < state->level; ++l) {
1974 324 : parity_ptr[l] = &parity[l];
1975 324 : ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
1976 324 : if (ret == -1) {
1977 1 : msg_status("No accessible %s file, only files will be checked.\n", lev_name(l));
1978 : /* continue anyway */
1979 1 : parity_ptr[l] = 0;
1980 : }
1981 : }
1982 : } else {
1983 : /* otherwise don't use any parity */
1984 28 : for (l = 0; l < state->level; ++l)
1985 22 : parity_ptr[l] = 0;
1986 : }
1987 :
1988 120 : if (fix)
1989 45 : msg_progress("Fixing...\n");
1990 75 : else if (!state->opt.auditonly)
1991 69 : msg_progress("Checking...\n");
1992 : else
1993 6 : msg_progress("Hashing...\n");
1994 :
1995 120 : error = 0;
1996 :
1997 : /* skip degenerated cases of empty parity, or skipping all */
1998 120 : if (blockstart < blockmax) {
1999 118 : ret = state_check_process(state, fix, parity_ptr, blockstart, blockmax);
2000 118 : if (ret == -1) {
2001 : /* LCOV_EXCL_START */
2002 : ++error;
2003 : /* continue, as we are already exiting */
2004 : /* LCOV_EXCL_STOP */
2005 : }
2006 : }
2007 :
2008 : /* try to close only if opened */
2009 601 : for (l = 0; l < state->level; ++l) {
2010 481 : if (parity_ptr[l]) {
2011 457 : ret = parity_close(parity_ptr[l]);
2012 : /* LCOV_EXCL_START */
2013 : if (ret == -1) {
2014 : log_fatal("DANGER! Unexpected close error in %s disk.\n", lev_name(l));
2015 : ++error;
2016 : /* continue, as we are already exiting */
2017 : }
2018 : /* LCOV_EXCL_STOP */
2019 : }
2020 : }
2021 :
2022 : /* abort if error are present */
2023 120 : if (error != 0)
2024 0 : return -1;
2025 120 : return 0;
2026 : }
2027 :
|