Line data Source code
1 : /*
2 : * Copyright (C) 2013 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "support.h"
21 : #include "elem.h"
22 : #include "state.h"
23 : #include "parity.h"
24 : #include "handle.h"
25 : #include "io.h"
26 : #include "raid/raid.h"
27 :
28 : /****************************************************************************/
29 : /* scrub */
30 :
31 0 : static const char* es(int err)
32 : {
33 0 : if (is_hw(err))
34 0 : return "error_io";
35 : else
36 0 : return "error";
37 : }
38 :
39 : /**
40 : * Buffer for storing the new hashes.
41 : */
42 : struct snapraid_rehash {
43 : unsigned char hash[HASH_MAX];
44 : struct snapraid_block* block;
45 : };
46 :
47 : /**
48 : * Scrub plan to use.
49 : */
50 : struct snapraid_plan {
51 : struct snapraid_state* state;
52 : int plan; /**< One of the SCRUB_*. */
53 : time_t timelimit; /**< Time limit. Valid only with SCRUB_AUTO. */
54 : block_off_t lastlimit; /**< Number of blocks allowed with time exactly at ::timelimit. Valid only with SCRUB_AUTO. */
55 : };
56 :
57 : /**
58 : * Check if we have to process the specified block index ::i.
59 : */
60 56145 : static int block_is_enabled(struct snapraid_plan* plan, block_off_t* countlast, block_off_t i)
61 : {
62 : time_t blocktime;
63 : snapraid_info info;
64 :
65 : /* don't scrub unused blocks in all plans */
66 56145 : info = info_get(&plan->state->infoarr, i);
67 56145 : if (info == 0)
68 0 : return 0;
69 :
70 : /* bad blocks are always scrubbed in all plans */
71 56145 : if (info_get_bad(info))
72 1772 : return 1;
73 :
74 54373 : switch (plan->plan) {
75 8522 : case SCRUB_FULL :
76 : /* in 'full' plan everything is scrubbed */
77 8522 : return 1;
78 9275 : case SCRUB_EVEN :
79 : /* in 'even' plan, scrub only even blocks */
80 9275 : return i % 2 == 0;
81 4687 : case SCRUB_NEW :
82 : /* in 'sync' plan, only blocks never scrubbed */
83 4687 : return info_get_justsynced(info);
84 3767 : case SCRUB_BAD :
85 : /* in 'bad' plan, only bad blocks (already reported) */
86 3767 : return 0;
87 : }
88 :
89 : /* if it's too new */
90 28122 : blocktime = info_get_time(info);
91 28122 : if (blocktime > plan->timelimit) {
92 : /* skip it */
93 4791 : return 0;
94 : }
95 :
96 : /* if the time is less than the limit, always include */
97 : /* otherwise, check if we reached the last limit count */
98 23331 : if (blocktime == plan->timelimit) {
99 : /* if we reached the count limit */
100 13957 : if (*countlast >= plan->lastlimit) {
101 : /* skip it */
102 12466 : return 0;
103 : }
104 :
105 1491 : ++*countlast;
106 : }
107 :
108 10865 : return 1;
109 : }
110 :
111 154782 : static void scrub_data_reader(struct snapraid_worker* worker, struct snapraid_task* task)
112 : {
113 154782 : struct snapraid_io* io = worker->io;
114 154782 : struct snapraid_state* state = io->state;
115 154782 : struct snapraid_handle* handle = worker->handle;
116 154782 : struct snapraid_disk* disk = handle->disk;
117 154782 : block_off_t blockcur = task->position;
118 154782 : unsigned char* buffer = task->buffer;
119 : int ret;
120 : char esc_buffer[ESC_MAX];
121 :
122 : /* if the disk position is not used */
123 154782 : if (!disk) {
124 : /* use an empty block */
125 0 : memset(buffer, 0, state->block_size);
126 0 : task->state = TASK_STATE_DONE;
127 1809 : return;
128 : }
129 :
130 : /* get the block */
131 154782 : task->block = fs_par2block_find(disk, blockcur);
132 :
133 : /* if the block is not used */
134 154782 : if (!block_has_file(task->block)) {
135 : /* use an empty block */
136 1809 : memset(buffer, 0, state->block_size);
137 1809 : task->state = TASK_STATE_DONE;
138 1809 : return;
139 : }
140 :
141 : /* get the file of this block */
142 152973 : task->file = fs_par2file_get(disk, blockcur, &task->file_pos);
143 :
144 : /* if the file is different than the current one, close it */
145 152973 : if (handle->file != 0 && handle->file != task->file) {
146 : /* keep a pointer at the file we are going to close for error reporting */
147 72508 : struct snapraid_file* report = handle->file;
148 72508 : ret = handle_close(handle);
149 72508 : if (ret == -1) {
150 : /* LCOV_EXCL_START */
151 : /* This one is really an unexpected error, because we are only reading */
152 : /* and closing a descriptor should never fail */
153 : log_tag("%s:%u:%s:%s: Close error. %s.\n", es(errno), blockcur, disk->name, esc_tag(report->sub, esc_buffer), strerror(errno));
154 : log_fatal_errno(errno, disk->name);
155 : log_fatal(errno, "Stopping at block %u\n", blockcur);
156 :
157 : if (is_hw(errno)) {
158 : task->state = TASK_STATE_IOERROR;
159 : } else {
160 : task->state = TASK_STATE_ERROR;
161 : }
162 : return;
163 : /* LCOV_EXCL_STOP */
164 : }
165 : }
166 :
167 152973 : ret = handle_open(handle, task->file, state->file_mode, log_error, 0); /* for missing file don't output a message */
168 152973 : if (ret == -1) {
169 0 : log_tag("%s:%u:%s:%s: Open error. %s.\n", es(errno), blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), strerror(errno));
170 0 : if (is_hw(errno)) {
171 : /* LCOV_EXCL_START */
172 : log_fatal_errno(errno, disk->name);
173 : log_fatal(errno, "Stopping at block %u\n", blockcur);
174 : task->state = TASK_STATE_IOERROR;
175 : return;
176 : /* LCOV_EXCL_STOP */
177 : }
178 :
179 0 : task->state = TASK_STATE_ERROR_CONTINUE;
180 0 : return;
181 : }
182 :
183 : /* check if the file is changed */
184 152973 : if (handle->st.st_size != task->file->size
185 152973 : || handle->st.st_mtime != task->file->mtime_sec
186 152973 : || STAT_NSEC(&handle->st) != task->file->mtime_nsec
187 : /* don't check the inode to support filesystem without persistent inodes */
188 : ) {
189 : /* report that the block and the file are not synced */
190 0 : task->is_timestamp_different = 1;
191 : /* follow */
192 : }
193 :
194 : /* note that we intentionally don't abort if the file has different attributes */
195 : /* from the last sync, as we are expected to return errors if running */
196 : /* in an unsynced array. This is just like the check command. */
197 :
198 152973 : task->read_size = handle_read(handle, task->file_pos, buffer, state->block_size, log_error, 0);
199 152973 : if (task->read_size == -1) {
200 0 : log_tag("%s:%u:%s:%s: Read error at position %u. %s.\n", es(errno), blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), task->file_pos, strerror(errno));
201 0 : if (is_hw(errno)) {
202 : /* LCOV_EXCL_START */
203 : log_error_errno(errno, disk->name);
204 : task->state = TASK_STATE_IOERROR_CONTINUE;
205 : return;
206 : /* LCOV_EXCL_STOP */
207 : }
208 :
209 0 : task->state = TASK_STATE_ERROR_CONTINUE;
210 0 : return;
211 : }
212 :
213 : /* store the path of the opened file */
214 152973 : pathcpy(task->path, sizeof(task->path), handle->path);
215 :
216 152973 : task->state = TASK_STATE_DONE;
217 : }
218 :
219 154782 : static void scrub_parity_reader(struct snapraid_worker* worker, struct snapraid_task* task)
220 : {
221 154782 : struct snapraid_io* io = worker->io;
222 154782 : struct snapraid_state* state = io->state;
223 154782 : struct snapraid_parity_handle* parity_handle = worker->parity_handle;
224 154782 : unsigned level = parity_handle->level;
225 154782 : block_off_t blockcur = task->position;
226 154782 : unsigned char* buffer = task->buffer;
227 : int ret;
228 :
229 : /* read the parity */
230 154782 : ret = parity_read(parity_handle, blockcur, buffer, state->block_size, log_error);
231 154782 : if (ret == -1) {
232 0 : log_tag("parity_%s:%u:%s: Read error. %s.\n", es(errno), blockcur, lev_config_name(level), strerror(errno));
233 0 : if (is_hw(errno)) {
234 : /* LCOV_EXCL_START */
235 : log_error_errno(errno, lev_config_name(level));
236 : task->state = TASK_STATE_IOERROR_CONTINUE;
237 : return;
238 : /* LCOV_EXCL_STOP */
239 : }
240 :
241 0 : task->state = TASK_STATE_ERROR_CONTINUE;
242 0 : return;
243 : }
244 :
245 154782 : task->state = TASK_STATE_DONE;
246 : }
247 :
248 11 : static int state_scrub_process(struct snapraid_state* state, struct snapraid_parity_handle* parity_handle, block_off_t blockstart, block_off_t blockmax, struct snapraid_plan* plan, time_t now)
249 : {
250 : struct snapraid_io io;
251 : struct snapraid_handle* handle;
252 : void* rehandle_alloc;
253 : struct snapraid_rehash* rehandle;
254 : unsigned diskmax;
255 : block_off_t blockcur;
256 : unsigned j;
257 : unsigned buffermax;
258 : data_off_t countsize;
259 : block_off_t countpos;
260 : block_off_t countmax;
261 : block_off_t countlast;
262 : block_off_t autosavedone;
263 : block_off_t autosavelimit;
264 : block_off_t autosavemissing;
265 : int ret;
266 : unsigned soft_error;
267 : unsigned silent_error;
268 : unsigned io_error;
269 : unsigned l;
270 : unsigned* waiting_map;
271 : unsigned waiting_mac;
272 : char esc_buffer[ESC_MAX];
273 : bit_vect_t* block_enabled;
274 :
275 : /* maps the disks to handles */
276 11 : handle = handle_mapping(state, &diskmax);
277 :
278 : /* rehash buffers */
279 11 : rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
280 :
281 : /* we need 1 * data + 2 * parity */
282 11 : buffermax = diskmax + 2 * state->level;
283 :
284 : /* initialize the io threads */
285 11 : io_init(&io, state, state->opt.io_cache, buffermax, scrub_data_reader, handle, diskmax, scrub_parity_reader, 0, parity_handle, state->level);
286 :
287 : /* possibly waiting disks */
288 11 : waiting_mac = diskmax > RAID_PARITY_MAX ? diskmax : RAID_PARITY_MAX;
289 11 : waiting_map = malloc_nofail(waiting_mac * sizeof(unsigned));
290 :
291 11 : soft_error = 0;
292 11 : silent_error = 0;
293 11 : io_error = 0;
294 :
295 11 : msg_progress("Selecting...\n");
296 :
297 : /* first count the number of blocks to process */
298 11 : countmax = 0;
299 11 : countlast = 0;
300 11 : block_enabled = calloc_nofail(1, bit_vect_size(blockmax)); /* preinitialize to 0 */
301 56156 : for (blockcur = blockstart; blockcur < blockmax; ++blockcur) {
302 56145 : if (!block_is_enabled(plan, &countlast, blockcur))
303 30348 : continue;
304 25797 : bit_vect_set(block_enabled, blockcur);
305 25797 : ++countmax;
306 : }
307 :
308 : /* compute the autosave size for all disk, even if not read */
309 : /* this makes sense because the speed should be almost the same */
310 : /* if the disks are read in parallel */
311 11 : autosavelimit = state->autosave / (diskmax * state->block_size);
312 11 : autosavemissing = countmax; /* blocks to do */
313 11 : autosavedone = 0; /* blocks done */
314 :
315 : /* drop until now */
316 11 : state_usage_waste(state);
317 :
318 11 : countsize = 0;
319 11 : countpos = 0;
320 :
321 11 : msg_progress("Scrubbing...\n");
322 :
323 : /* start all the worker threads */
324 11 : io_start(&io, blockstart, blockmax, block_enabled);
325 :
326 11 : int alert = state_progress_begin(state, blockstart, blockmax, countmax);
327 11 : if (alert > 0)
328 0 : goto end;
329 11 : if (alert < 0)
330 0 : goto bail;
331 :
332 25797 : while (1) {
333 : unsigned char* buffer_recov[LEV_MAX];
334 : snapraid_info info;
335 : int error_on_this_block;
336 : int silent_error_on_this_block;
337 : int io_error_on_this_block;
338 : int block_is_unsynced;
339 : int rehash;
340 : void** buffer;
341 :
342 : /* go to the next block */
343 25808 : blockcur = io_read_next(&io, &buffer);
344 25808 : if (blockcur >= blockmax)
345 11 : break;
346 :
347 : /* until now is scheduling */
348 25797 : state_usage_sched(state);
349 :
350 : /* one more block processed for autosave */
351 25797 : ++autosavedone;
352 25797 : --autosavemissing;
353 :
354 : /* by default process the block, and skip it if something goes wrong */
355 25797 : error_on_this_block = 0;
356 25797 : silent_error_on_this_block = 0;
357 25797 : io_error_on_this_block = 0;
358 :
359 : /* if all the blocks at this address are synced */
360 : /* if not, parity is not even checked */
361 25797 : block_is_unsynced = 0;
362 :
363 : /* get block specific info */
364 25797 : info = info_get(&state->infoarr, blockcur);
365 :
366 : /* if we have to use the old hash */
367 25797 : rehash = info_get_rehash(info);
368 :
369 : /* for each disk, process the block */
370 180579 : for (j = 0; j < diskmax; ++j) {
371 : struct snapraid_task* task;
372 : int read_size;
373 : unsigned char hash[HASH_MAX];
374 : struct snapraid_block* block;
375 : int file_is_unsynced;
376 : struct snapraid_disk* disk;
377 : struct snapraid_file* file;
378 : block_off_t file_pos;
379 : unsigned diskcur;
380 :
381 : /* if the file on this disk is synced */
382 : /* if not, silent errors are assumed as expected error */
383 154782 : file_is_unsynced = 0;
384 :
385 : /* until now is misc */
386 154782 : state_usage_misc(state);
387 :
388 : /* get the next task */
389 154782 : task = io_data_read(&io, &diskcur, waiting_map, &waiting_mac);
390 :
391 : /* until now is disk */
392 154782 : state_usage_disk(state, handle, waiting_map, waiting_mac);
393 :
394 : /* get the task results */
395 154782 : disk = task->disk;
396 154782 : block = task->block;
397 154782 : file = task->file;
398 154782 : file_pos = task->file_pos;
399 154782 : read_size = task->read_size;
400 :
401 : /* by default no rehash in case of "continue" */
402 154782 : rehandle[diskcur].block = 0;
403 :
404 : /* if the disk position is not used */
405 154782 : if (!disk)
406 2729 : continue;
407 :
408 154782 : state_usage_file(state, disk, file);
409 :
410 : /* if the block is unsynced, errors are expected */
411 154782 : if (block_has_invalid_parity(block)) {
412 : /* report that the block and the file are not synced */
413 0 : block_is_unsynced = 1;
414 0 : file_is_unsynced = 1;
415 : /* follow */
416 : }
417 :
418 : /* if the block is not used */
419 154782 : if (!block_has_file(block))
420 1809 : continue;
421 :
422 : /* if the block is unsynced, errors are expected */
423 152973 : if (task->is_timestamp_different) {
424 : /* report that the block and the file are not synced */
425 0 : block_is_unsynced = 1;
426 0 : file_is_unsynced = 1;
427 : /* follow */
428 : }
429 :
430 : /* handle error conditions */
431 152973 : if (task->state == TASK_STATE_IOERROR) {
432 : /* LCOV_EXCL_START */
433 : ++io_error;
434 : goto bail;
435 : /* LCOV_EXCL_STOP */
436 : }
437 152973 : if (task->state == TASK_STATE_ERROR) {
438 : /* LCOV_EXCL_START */
439 : ++soft_error;
440 : goto bail;
441 : /* LCOV_EXCL_STOP */
442 : }
443 152973 : if (task->state == TASK_STATE_ERROR_CONTINUE) {
444 0 : ++soft_error;
445 0 : error_on_this_block = 1;
446 0 : continue;
447 : }
448 152973 : if (task->state == TASK_STATE_IOERROR_CONTINUE) {
449 0 : ++io_error;
450 0 : if (io_error >= state->opt.io_error_limit) {
451 : /* LCOV_EXCL_START */
452 : log_fatal(EIO, "DANGER! Too many input/output errors in the %s disk. It isn't possible to continue.\n", disk->dir);
453 : log_fatal(EIO, "Stopping at block %u\n", blockcur);
454 : goto bail;
455 : /* LCOV_EXCL_STOP */
456 : }
457 :
458 : /* otherwise continue */
459 0 : io_error_on_this_block = 1;
460 0 : continue;
461 : }
462 152973 : if (task->state != TASK_STATE_DONE) {
463 : /* LCOV_EXCL_START */
464 : log_fatal(EINTERNAL, "Internal inconsistency in task state\n");
465 : os_abort();
466 : /* LCOV_EXCL_STOP */
467 : }
468 :
469 152973 : countsize += read_size;
470 :
471 : /* now compute the hash */
472 152973 : if (rehash) {
473 27203 : memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
474 :
475 : /* compute the new hash, and store it */
476 27203 : rehandle[diskcur].block = block;
477 27203 : memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
478 : } else {
479 125770 : memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
480 : }
481 :
482 : /* until now is hash */
483 152973 : state_usage_hash(state);
484 :
485 152973 : if (block_has_updated_hash(block)) {
486 : /* compare the hash */
487 152973 : if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) {
488 920 : unsigned diff = memdiff(hash, block->hash, BLOCK_HASH_SIZE);
489 :
490 : /* it's a silent error only if we are dealing with synced files */
491 920 : if (file_is_unsynced) {
492 0 : log_tag("error:%u:%s:%s: Data error at position %u, diff hash bits %u/%u\n", blockcur, disk->name, esc_tag(file->sub, esc_buffer), file_pos, diff, BLOCK_HASH_SIZE * 8);
493 0 : ++soft_error;
494 0 : error_on_this_block = 1;
495 : } else {
496 920 : log_tag("error_data:%u:%s:%s: Data error at position %u, diff hash bits %u/%u\n", blockcur, disk->name, esc_tag(file->sub, esc_buffer), file_pos, diff, BLOCK_HASH_SIZE * 8);
497 920 : log_error(EDATA, "Data error in file '%s' at position '%u', diff hash bits %u/%u\n", task->path, file_pos, diff, BLOCK_HASH_SIZE * 8);
498 920 : ++silent_error;
499 920 : silent_error_on_this_block = 1;
500 : }
501 920 : continue;
502 : }
503 : }
504 : }
505 :
506 : /* buffers for parity read and not computed */
507 180579 : for (l = 0; l < state->level; ++l)
508 154782 : buffer_recov[l] = buffer[diskmax + state->level + l];
509 25797 : for (; l < LEV_MAX; ++l)
510 0 : buffer_recov[l] = 0;
511 :
512 : /* until now is misc */
513 25797 : state_usage_misc(state);
514 :
515 : /* read the parity */
516 180579 : for (l = 0; l < state->level; ++l) {
517 : struct snapraid_task* task;
518 : unsigned levcur;
519 :
520 154782 : task = io_parity_read(&io, &levcur, waiting_map, &waiting_mac);
521 :
522 : /* until now is parity */
523 154782 : state_usage_parity(state, waiting_map, waiting_mac);
524 :
525 : /* handle error conditions */
526 154782 : if (task->state == TASK_STATE_IOERROR) {
527 : /* LCOV_EXCL_START */
528 : ++io_error;
529 : goto bail;
530 : /* LCOV_EXCL_STOP */
531 : }
532 154782 : if (task->state == TASK_STATE_ERROR) {
533 : /* LCOV_EXCL_START */
534 : ++soft_error;
535 : goto bail;
536 : /* LCOV_EXCL_STOP */
537 : }
538 154782 : if (task->state == TASK_STATE_ERROR_CONTINUE) {
539 0 : ++soft_error;
540 0 : error_on_this_block = 1;
541 :
542 : /* if continuing on error, clear the missing buffer */
543 0 : buffer_recov[levcur] = 0;
544 0 : continue;
545 : }
546 154782 : if (task->state == TASK_STATE_IOERROR_CONTINUE) {
547 0 : ++io_error;
548 0 : if (io_error >= state->opt.io_error_limit) {
549 : /* LCOV_EXCL_START */
550 : log_fatal(EIO, "DANGER! Too many input/output errors in the %s disk. It isn't possible to continue.\n", lev_name(levcur));
551 : log_fatal(EIO, "Stopping at block %u\n", blockcur);
552 : goto bail;
553 : /* LCOV_EXCL_STOP */
554 : }
555 :
556 : /* otherwise continue */
557 0 : io_error_on_this_block = 1;
558 :
559 : /* if continuing on error, clear the missing buffer */
560 0 : buffer_recov[levcur] = 0;
561 0 : continue;
562 : }
563 154782 : if (task->state != TASK_STATE_DONE) {
564 : /* LCOV_EXCL_START */
565 : log_fatal(EINTERNAL, "Internal inconsistency in task state\n");
566 : os_abort();
567 : /* LCOV_EXCL_STOP */
568 : }
569 : }
570 :
571 : /* if we have read all the data required and it's correct, proceed with the parity check */
572 25797 : if (!error_on_this_block && !silent_error_on_this_block && !io_error_on_this_block) {
573 :
574 : /* compute the parity */
575 24877 : raid_gen(diskmax, state->level, state->block_size, buffer);
576 :
577 : /* compare the parity */
578 174139 : for (l = 0; l < state->level; ++l) {
579 149262 : if (buffer_recov[l] && memcmp(buffer[diskmax + l], buffer_recov[l], state->block_size) != 0) {
580 0 : unsigned diff = memdiff(buffer[diskmax + l], buffer_recov[l], state->block_size);
581 :
582 : /* it's a silent error only if we are dealing with synced blocks */
583 0 : if (block_is_unsynced) {
584 0 : log_tag("parity_error:%u:%s: Data error, diff parity bits %u/%u\n", blockcur, lev_config_name(l), diff, state->block_size * 8);
585 0 : ++soft_error;
586 0 : error_on_this_block = 1;
587 : } else {
588 0 : log_tag("parity_error_data:%u:%s: Data error, diff parity bits %u/%u\n", blockcur, lev_config_name(l), diff, state->block_size * 8);
589 0 : log_error(EDATA, "Data error in parity '%s' at position '%u', diff parity bits %u/%u\n", lev_config_name(l), blockcur, diff, state->block_size * 8);
590 0 : ++silent_error;
591 0 : silent_error_on_this_block = 1;
592 : }
593 : }
594 : }
595 :
596 : /* until now is raid */
597 24877 : state_usage_raid(state);
598 : }
599 :
600 25797 : if (silent_error_on_this_block || io_error_on_this_block) {
601 : /* set the error status keeping other info */
602 920 : info_set(&state->infoarr, blockcur, info_set_bad(info));
603 24877 : } else if (error_on_this_block) {
604 : /* do nothing, as this is a generic error */
605 : /* likely caused by a not synced array */
606 : } else {
607 : /* if rehash is needed */
608 24877 : if (rehash) {
609 : /* store all the new hash already computed */
610 32466 : for (j = 0; j < diskmax; ++j) {
611 27828 : if (rehandle[j].block)
612 27203 : memcpy(rehandle[j].block->hash, rehandle[j].hash, BLOCK_HASH_SIZE);
613 : }
614 : }
615 :
616 : /* update the time info of the block */
617 : /* and clear any other flag */
618 24877 : info_set(&state->infoarr, blockcur, info_make(now, 0, 0, 0));
619 : }
620 :
621 : /* mark the state as needing write */
622 25797 : state->need_write = 1;
623 :
624 : /* count the number of processed block */
625 25797 : ++countpos;
626 :
627 : /* progress */
628 25797 : if (state_progress(state, &io, blockcur, countpos, countmax, countsize)) {
629 : /* LCOV_EXCL_START */
630 : break;
631 : /* LCOV_EXCL_STOP */
632 : }
633 :
634 : /* thermal control */
635 25797 : if (state_thermal_alarm(state)) {
636 : /* until now is misc */
637 0 : state_usage_misc(state);
638 :
639 0 : state_progress_stop(state);
640 :
641 0 : state_thermal_cooldown(state);
642 :
643 0 : state_progress_restart(state);
644 :
645 : /* drop until now */
646 0 : state_usage_waste(state);
647 : }
648 :
649 : /* autosave */
650 25797 : if (state->autosave != 0
651 0 : && autosavedone >= autosavelimit /* if we have reached the limit */
652 0 : && autosavemissing >= autosavelimit /* if we have at least a full step to do */
653 : ) {
654 0 : autosavedone = 0; /* restart the counter */
655 :
656 : /* until now is misc */
657 0 : state_usage_misc(state);
658 :
659 0 : state_progress_stop(state);
660 :
661 0 : msg_progress("Autosaving...\n");
662 0 : state_write(state);
663 :
664 0 : state_progress_restart(state);
665 :
666 : /* drop until now */
667 0 : state_usage_waste(state);
668 : }
669 : }
670 :
671 11 : end:
672 11 : state_progress_end(state, countpos, countmax, countsize, "Nothing to scrub. Use the -p PLAN option to select a different plan, like -p full.\n");
673 :
674 : /* save the new state if required */
675 11 : if (state->need_write || state->opt.force_content_write)
676 9 : state_write(state);
677 :
678 11 : state_usage_print(state);
679 :
680 11 : if (soft_error || silent_error || io_error) {
681 1 : msg_status("\n");
682 1 : msg_status("%8u soft errors\n", soft_error);
683 1 : msg_status("%8u io errors\n", io_error);
684 1 : msg_status("%8u data errors\n", silent_error);
685 : } else {
686 10 : msg_status("Everything OK\n");
687 : }
688 :
689 11 : if (soft_error)
690 0 : log_fatal(ESOFT, "WARNING! Unexpected soft errors!\n");
691 11 : if (io_error)
692 0 : log_fatal(EIO, "DANGER! Unexpected input/output errors! The failing blocks are now marked as bad!\n");
693 11 : if (silent_error)
694 1 : log_fatal(EDATA, "DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n");
695 11 : if (io_error || silent_error) {
696 1 : log_fatal(ESOFT, "Use 'snapraid status' to list the bad blocks.\n");
697 1 : log_fatal(ESOFT, "Use 'snapraid -e fix' to recover them.\n");
698 1 : log_fatal(ESOFT, "Use 'snapraid -p bad scrub' to recheck after fixing to clear the bad state.\n");
699 : }
700 :
701 11 : log_tag("summary:error_soft:%u\n", soft_error);
702 11 : log_tag("summary:error_io:%u\n", io_error);
703 11 : log_tag("summary:error_data:%u\n", silent_error);
704 11 : if (soft_error + silent_error + io_error == 0)
705 10 : log_tag("summary:exit:ok\n");
706 1 : else if (silent_error + io_error == 0)
707 0 : log_tag("summary:exit:warning\n");
708 : else
709 1 : log_tag("summary:exit:error\n");
710 11 : log_flush();
711 :
712 11 : bail:
713 : /* stop all the worker threads */
714 11 : io_stop(&io);
715 :
716 77 : for (j = 0; j < diskmax; ++j) {
717 66 : struct snapraid_file* file = handle[j].file;
718 66 : struct snapraid_disk* disk = handle[j].disk;
719 66 : ret = handle_close(&handle[j]);
720 66 : if (ret == -1) {
721 : /* LCOV_EXCL_START */
722 : log_tag("%s:%u:%s:%s: Close error. %s.\n", es(errno), blockcur, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
723 : log_fatal_errno(errno, disk->name);
724 :
725 : if (is_hw(errno)) {
726 : ++io_error;
727 : } else {
728 : ++soft_error;
729 : }
730 : /* continue, as we are already exiting */
731 : /* LCOV_EXCL_STOP */
732 : }
733 : }
734 :
735 11 : free(handle);
736 11 : free(rehandle_alloc);
737 11 : free(waiting_map);
738 11 : io_done(&io);
739 11 : free(block_enabled);
740 :
741 11 : if (state->opt.expect_recoverable) {
742 1 : if (soft_error + silent_error + io_error == 0)
743 0 : return -1;
744 : } else {
745 10 : if (soft_error + silent_error + io_error != 0)
746 0 : return -1;
747 : }
748 :
749 11 : if (alert < 0)
750 0 : return -1;
751 :
752 11 : return 0;
753 : }
754 :
755 : /**
756 : * Return a * b / c approximated to the upper value.
757 : */
758 2 : static uint32_t md(uint32_t a, uint32_t b, uint32_t c)
759 : {
760 2 : uint64_t v = a;
761 :
762 2 : v *= b;
763 2 : v += c - 1;
764 2 : v /= c;
765 :
766 2 : return v;
767 : }
768 :
769 11 : int state_scrub(struct snapraid_state* state, int plan100, int olderthan)
770 : {
771 : block_off_t blockmax;
772 : block_off_t countlimit;
773 : block_off_t count;
774 : time_t recentlimit;
775 : int ret;
776 : struct snapraid_parity_handle parity_handle[LEV_MAX];
777 : struct snapraid_plan ps;
778 : unsigned process_error;
779 : time_t now;
780 : unsigned l;
781 :
782 : /* get the present time */
783 11 : now = time(0);
784 :
785 11 : msg_progress("Initializing...\n");
786 :
787 11 : if ((plan100 == SCRUB_BAD || plan100 == SCRUB_NEW || plan100 == SCRUB_FULL)
788 4 : && olderthan >= 0) {
789 : /* LCOV_EXCL_START */
790 : log_fatal(EUSER, "You can specify -o, --older-than only with a numeric percentage.\n");
791 : exit(EXIT_FAILURE);
792 : /* LCOV_EXCL_STOP */
793 : }
794 :
795 11 : blockmax = parity_allocated_size(state);
796 :
797 : /* preinitialize to avoid warnings */
798 11 : countlimit = 0;
799 11 : recentlimit = 0;
800 :
801 11 : ps.state = state;
802 11 : if (state->opt.force_scrub_even) {
803 1 : ps.plan = SCRUB_EVEN;
804 10 : } else if (plan100 == SCRUB_FULL) {
805 2 : ps.plan = SCRUB_FULL;
806 2 : msg_progress("Scrub plan: full. All data blocks will be checked.\n");
807 8 : } else if (plan100 == SCRUB_NEW) {
808 1 : ps.plan = SCRUB_NEW;
809 1 : msg_progress("Scrub plan: new. Only blocks that have never been scrubbed will be checked.\n");
810 7 : } else if (plan100 == SCRUB_BAD) {
811 1 : ps.plan = SCRUB_BAD;
812 1 : msg_progress("Scrub plan: bad. Only blocks previously marked as bad will be checked.\n");
813 6 : } else if (state->opt.force_scrub_at) {
814 : /* scrub the specified amount of blocks */
815 4 : ps.plan = SCRUB_AUTO;
816 4 : countlimit = state->opt.force_scrub_at;
817 4 : recentlimit = now;
818 : } else {
819 2 : ps.plan = SCRUB_AUTO;
820 2 : if (plan100 >= 0) {
821 1 : countlimit = md(blockmax, plan100, 10000);
822 : } else {
823 : /* by default scrub 8.33% of the array (100/12=8.(3)) */
824 1 : countlimit = md(blockmax, 1, 12);
825 : }
826 :
827 2 : if (olderthan >= 0) {
828 1 : recentlimit = now - olderthan * 24 * 3600;
829 : } else {
830 : /* by default use a 10 day time limit */
831 1 : recentlimit = now - 10 * 24 * 3600;
832 : }
833 :
834 2 : if (plan100 >= 0) {
835 1 : if (olderthan >= 0)
836 0 : msg_progress("Scrub plan: auto. %.1f%% of the array, older than %d days, will be checked.\n", plan100 / 100.0, olderthan);
837 : else
838 1 : msg_progress("Scrub plan: auto. %.1f%% of the array, older than 10 days, will be checked.\n", plan100 / 100.0);
839 : } else {
840 1 : if (olderthan >= 0)
841 1 : msg_progress("Scrub plan: auto. 8.3%% of the array, older than %d days, will be checked.\n", olderthan);
842 : else
843 0 : msg_progress("Scrub plan: auto. 8.3%% of the array, older than 10 days, will be checked.\n");
844 : }
845 : }
846 :
847 11 : count = 0;
848 32 : for (tommy_node* j = tommy_list_head(&state->bucketlist); j != 0; j = j->next) {
849 21 : struct snapraid_bucket* bucket = j->data;
850 21 : count += bucket->count_scrubbed + bucket->count_justsynced;
851 : }
852 :
853 11 : if (!count) {
854 : /* LCOV_EXCL_START */
855 : log_fatal(EUSER, "The array is empty.\n");
856 : exit(EXIT_FAILURE);
857 : /* LCOV_EXCL_STOP */
858 : }
859 :
860 : /* compute the limits from count/recentlimit */
861 11 : if (ps.plan == SCRUB_AUTO) {
862 : /* no more than the full count */
863 6 : if (countlimit > count)
864 2 : countlimit = count;
865 :
866 : /* by default process everything */
867 6 : ps.timelimit = now;
868 6 : ps.lastlimit = 0;
869 :
870 6 : tommy_node* j = tommy_list_head(&state->bucketlist);
871 6 : block_off_t processed_count = 0;
872 11 : while (j) {
873 9 : struct snapraid_bucket* bucket = j->data;
874 9 : block_off_t bucket_count = bucket->count_justsynced + bucket->count_scrubbed;
875 :
876 9 : if (bucket->time_at > recentlimit) {
877 1 : ps.timelimit = recentlimit;
878 1 : ps.lastlimit = 0;
879 1 : break;
880 : }
881 :
882 8 : if (processed_count + bucket_count > countlimit) {
883 3 : ps.timelimit = bucket->time_at;
884 3 : ps.lastlimit = countlimit - processed_count;
885 3 : processed_count = countlimit;
886 3 : break;
887 : }
888 :
889 5 : processed_count += bucket_count;
890 5 : j = j->next;
891 : }
892 :
893 : /* if nothing to scrub, disable also other limits */
894 6 : if (processed_count == 0) {
895 1 : ps.timelimit = 0;
896 1 : ps.lastlimit = 0;
897 : }
898 :
899 6 : log_tag("count_limit:%u\n", countlimit);
900 6 : log_tag("time_limit:%" PRIu64 "\n", (uint64_t)ps.timelimit);
901 6 : log_tag("last_limit:%u\n", ps.lastlimit);
902 : } else {
903 : /* avoid compiler warnings */
904 5 : ps.timelimit = 0;
905 5 : ps.lastlimit = 0;
906 : }
907 :
908 : /* open the file for reading */
909 77 : for (l = 0; l < state->level; ++l) {
910 66 : ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
911 66 : if (ret == -1) {
912 : /* LCOV_EXCL_START */
913 : log_tag("parity_%s:%u:%s: Open error. %s.\n", es(errno), blockmax, lev_config_name(l), strerror(errno));
914 : log_fatal_errno(errno, lev_config_name(l));
915 : exit(EXIT_FAILURE);
916 : /* LCOV_EXCL_STOP */
917 : }
918 : }
919 :
920 11 : process_error = 0;
921 :
922 11 : ret = state_scrub_process(state, parity_handle, 0, blockmax, &ps, now);
923 11 : if (ret == -1) {
924 0 : ++process_error;
925 : /* continue, as we are already exiting */
926 : }
927 :
928 77 : for (l = 0; l < state->level; ++l) {
929 66 : ret = parity_close(&parity_handle[l]);
930 66 : if (ret == -1) {
931 : /* LCOV_EXCL_START */
932 : log_tag("parity_%s:%u:%s: Close error. %s.\n", es(errno), blockmax, lev_config_name(l), strerror(errno));
933 : log_fatal_errno(errno, lev_config_name(l));
934 :
935 : ++process_error;
936 : /* continue, as we are already exiting */
937 : /* LCOV_EXCL_STOP */
938 : }
939 : }
940 :
941 11 : if (process_error != 0)
942 0 : return -1;
943 11 : return 0;
944 : }
945 :
|