LCOV - code coverage report
Current view: top level - cmdline - scrub.c (source / functions) Hit Total Coverage
Test: lcov.info Lines: 303 382 79.3 %
Date: 2026-03-15 15:58:19 Functions: 6 7 85.7 %

          Line data    Source code
       1             : /*
       2             :  * Copyright (C) 2013 Andrea Mazzoleni
       3             :  *
       4             :  * This program is free software: you can redistribute it and/or modify
       5             :  * it under the terms of the GNU General Public License as published by
       6             :  * the Free Software Foundation, either version 3 of the License, or
       7             :  * (at your option) any later version.
       8             :  *
       9             :  * This program is distributed in the hope that it will be useful,
      10             :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      12             :  * GNU General Public License for more details.
      13             :  *
      14             :  * You should have received a copy of the GNU General Public License
      15             :  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "portable.h"
      19             : 
      20             : #include "support.h"
      21             : #include "elem.h"
      22             : #include "state.h"
      23             : #include "parity.h"
      24             : #include "handle.h"
      25             : #include "io.h"
      26             : #include "raid/raid.h"
      27             : 
      28             : /****************************************************************************/
      29             : /* scrub */
      30             : 
      31           0 : static const char* es(int err)
      32             : {
      33           0 :         if (is_hw(err))
      34           0 :                 return "error_io";
      35             :         else
      36           0 :                 return "error";
      37             : }
      38             : 
      39             : /**
      40             :  * Buffer for storing the new hashes.
      41             :  */
      42             : struct snapraid_rehash {
      43             :         unsigned char hash[HASH_MAX];
      44             :         struct snapraid_block* block;
      45             : };
      46             : 
      47             : /**
      48             :  * Scrub plan to use.
      49             :  */
      50             : struct snapraid_plan {
      51             :         struct snapraid_state* state;
      52             :         int plan; /**< One of the SCRUB_*. */
      53             :         time_t timelimit; /**< Time limit. Valid only with SCRUB_AUTO. */
      54             :         block_off_t lastlimit; /**< Number of blocks allowed with time exactly at ::timelimit. Valid only with SCRUB_AUTO. */
      55             : };
      56             : 
      57             : /**
      58             :  * Check if we have to process the specified block index ::i.
      59             :  */
      60       56145 : static int block_is_enabled(struct snapraid_plan* plan, block_off_t* countlast, block_off_t i)
      61             : {
      62             :         time_t blocktime;
      63             :         snapraid_info info;
      64             : 
      65             :         /* don't scrub unused blocks in all plans */
      66       56145 :         info = info_get(&plan->state->infoarr, i);
      67       56145 :         if (info == 0)
      68           0 :                 return 0;
      69             : 
      70             :         /* bad blocks are always scrubbed in all plans */
      71       56145 :         if (info_get_bad(info))
      72        1772 :                 return 1;
      73             : 
      74       54373 :         switch (plan->plan) {
      75        8522 :         case SCRUB_FULL :
      76             :                 /* in 'full' plan everything is scrubbed */
      77        8522 :                 return 1;
      78        9275 :         case SCRUB_EVEN :
      79             :                 /* in 'even' plan, scrub only even blocks */
      80        9275 :                 return i % 2 == 0;
      81        4687 :         case SCRUB_NEW :
      82             :                 /* in 'sync' plan, only blocks never scrubbed */
      83        4687 :                 return info_get_justsynced(info);
      84        3767 :         case SCRUB_BAD :
      85             :                 /* in 'bad' plan, only bad blocks (already reported) */
      86        3767 :                 return 0;
      87             :         }
      88             : 
      89             :         /* if it's too new */
      90       28122 :         blocktime = info_get_time(info);
      91       28122 :         if (blocktime > plan->timelimit) {
      92             :                 /* skip it */
      93        4791 :                 return 0;
      94             :         }
      95             : 
      96             :         /* if the time is less than the limit, always include */
      97             :         /* otherwise, check if we reached the last limit count */
      98       23331 :         if (blocktime == plan->timelimit) {
      99             :                 /* if we reached the count limit */
     100       13957 :                 if (*countlast >= plan->lastlimit) {
     101             :                         /* skip it */
     102       12466 :                         return 0;
     103             :                 }
     104             : 
     105        1491 :                 ++*countlast;
     106             :         }
     107             : 
     108       10865 :         return 1;
     109             : }
     110             : 
     111      154782 : static void scrub_data_reader(struct snapraid_worker* worker, struct snapraid_task* task)
     112             : {
     113      154782 :         struct snapraid_io* io = worker->io;
     114      154782 :         struct snapraid_state* state = io->state;
     115      154782 :         struct snapraid_handle* handle = worker->handle;
     116      154782 :         struct snapraid_disk* disk = handle->disk;
     117      154782 :         block_off_t blockcur = task->position;
     118      154782 :         unsigned char* buffer = task->buffer;
     119             :         int ret;
     120             :         char esc_buffer[ESC_MAX];
     121             : 
     122             :         /* if the disk position is not used */
     123      154782 :         if (!disk) {
     124             :                 /* use an empty block */
     125           0 :                 memset(buffer, 0, state->block_size);
     126           0 :                 task->state = TASK_STATE_DONE;
     127        1809 :                 return;
     128             :         }
     129             : 
     130             :         /* get the block */
     131      154782 :         task->block = fs_par2block_find(disk, blockcur);
     132             : 
     133             :         /* if the block is not used */
     134      154782 :         if (!block_has_file(task->block)) {
     135             :                 /* use an empty block */
     136        1809 :                 memset(buffer, 0, state->block_size);
     137        1809 :                 task->state = TASK_STATE_DONE;
     138        1809 :                 return;
     139             :         }
     140             : 
     141             :         /* get the file of this block */
     142      152973 :         task->file = fs_par2file_get(disk, blockcur, &task->file_pos);
     143             : 
     144             :         /* if the file is different than the current one, close it */
     145      152973 :         if (handle->file != 0 && handle->file != task->file) {
     146             :                 /* keep a pointer at the file we are going to close for error reporting */
     147       72508 :                 struct snapraid_file* report = handle->file;
     148       72508 :                 ret = handle_close(handle);
     149       72508 :                 if (ret == -1) {
     150             :                         /* LCOV_EXCL_START */
     151             :                         /* This one is really an unexpected error, because we are only reading */
     152             :                         /* and closing a descriptor should never fail */
     153             :                         log_tag("%s:%u:%s:%s: Close error. %s.\n", es(errno), blockcur, disk->name, esc_tag(report->sub, esc_buffer), strerror(errno));
     154             :                         log_fatal_errno(errno, disk->name);
     155             :                         log_fatal(errno, "Stopping at block %u\n", blockcur);
     156             : 
     157             :                         if (is_hw(errno)) {
     158             :                                 task->state = TASK_STATE_IOERROR;
     159             :                         } else {
     160             :                                 task->state = TASK_STATE_ERROR;
     161             :                         }
     162             :                         return;
     163             :                         /* LCOV_EXCL_STOP */
     164             :                 }
     165             :         }
     166             : 
     167      152973 :         ret = handle_open(handle, task->file, state->file_mode, log_error, 0); /* for missing file don't output a message */
     168      152973 :         if (ret == -1) {
     169           0 :                 log_tag("%s:%u:%s:%s: Open error. %s.\n", es(errno), blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), strerror(errno));
     170           0 :                 if (is_hw(errno)) {
     171             :                         /* LCOV_EXCL_START */
     172             :                         log_fatal_errno(errno, disk->name);
     173             :                         log_fatal(errno, "Stopping at block %u\n", blockcur);
     174             :                         task->state = TASK_STATE_IOERROR;
     175             :                         return;
     176             :                         /* LCOV_EXCL_STOP */
     177             :                 }
     178             : 
     179           0 :                 task->state = TASK_STATE_ERROR_CONTINUE;
     180           0 :                 return;
     181             :         }
     182             : 
     183             :         /* check if the file is changed */
     184      152973 :         if (handle->st.st_size != task->file->size
     185      152973 :                 || handle->st.st_mtime != task->file->mtime_sec
     186      152973 :                 || STAT_NSEC(&handle->st) != task->file->mtime_nsec
     187             :                 /* don't check the inode to support filesystem without persistent inodes */
     188             :         ) {
     189             :                 /* report that the block and the file are not synced */
     190           0 :                 task->is_timestamp_different = 1;
     191             :                 /* follow */
     192             :         }
     193             : 
     194             :         /* note that we intentionally don't abort if the file has different attributes */
     195             :         /* from the last sync, as we are expected to return errors if running */
     196             :         /* in an unsynced array. This is just like the check command. */
     197             : 
     198      152973 :         task->read_size = handle_read(handle, task->file_pos, buffer, state->block_size, log_error, 0);
     199      152973 :         if (task->read_size == -1) {
     200           0 :                 log_tag("%s:%u:%s:%s: Read error at position %u. %s.\n", es(errno), blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), task->file_pos, strerror(errno));
     201           0 :                 if (is_hw(errno)) {
     202             :                         /* LCOV_EXCL_START */
     203             :                         log_error_errno(errno, disk->name);
     204             :                         task->state = TASK_STATE_IOERROR_CONTINUE;
     205             :                         return;
     206             :                         /* LCOV_EXCL_STOP */
     207             :                 }
     208             : 
     209           0 :                 task->state = TASK_STATE_ERROR_CONTINUE;
     210           0 :                 return;
     211             :         }
     212             : 
     213             :         /* store the path of the opened file */
     214      152973 :         pathcpy(task->path, sizeof(task->path), handle->path);
     215             : 
     216      152973 :         task->state = TASK_STATE_DONE;
     217             : }
     218             : 
     219      154782 : static void scrub_parity_reader(struct snapraid_worker* worker, struct snapraid_task* task)
     220             : {
     221      154782 :         struct snapraid_io* io = worker->io;
     222      154782 :         struct snapraid_state* state = io->state;
     223      154782 :         struct snapraid_parity_handle* parity_handle = worker->parity_handle;
     224      154782 :         unsigned level = parity_handle->level;
     225      154782 :         block_off_t blockcur = task->position;
     226      154782 :         unsigned char* buffer = task->buffer;
     227             :         int ret;
     228             : 
     229             :         /* read the parity */
     230      154782 :         ret = parity_read(parity_handle, blockcur, buffer, state->block_size, log_error);
     231      154782 :         if (ret == -1) {
     232           0 :                 log_tag("parity_%s:%u:%s: Read error. %s.\n", es(errno), blockcur, lev_config_name(level), strerror(errno));
     233           0 :                 if (is_hw(errno)) {
     234             :                         /* LCOV_EXCL_START */
     235             :                         log_error_errno(errno, lev_config_name(level));
     236             :                         task->state = TASK_STATE_IOERROR_CONTINUE;
     237             :                         return;
     238             :                         /* LCOV_EXCL_STOP */
     239             :                 }
     240             : 
     241           0 :                 task->state = TASK_STATE_ERROR_CONTINUE;
     242           0 :                 return;
     243             :         }
     244             : 
     245      154782 :         task->state = TASK_STATE_DONE;
     246             : }
     247             : 
     248          11 : static int state_scrub_process(struct snapraid_state* state, struct snapraid_parity_handle* parity_handle, block_off_t blockstart, block_off_t blockmax, struct snapraid_plan* plan, time_t now)
     249             : {
     250             :         struct snapraid_io io;
     251             :         struct snapraid_handle* handle;
     252             :         void* rehandle_alloc;
     253             :         struct snapraid_rehash* rehandle;
     254             :         unsigned diskmax;
     255             :         block_off_t blockcur;
     256             :         unsigned j;
     257             :         unsigned buffermax;
     258             :         data_off_t countsize;
     259             :         block_off_t countpos;
     260             :         block_off_t countmax;
     261             :         block_off_t countlast;
     262             :         block_off_t autosavedone;
     263             :         block_off_t autosavelimit;
     264             :         block_off_t autosavemissing;
     265             :         int ret;
     266             :         unsigned soft_error;
     267             :         unsigned silent_error;
     268             :         unsigned io_error;
     269             :         unsigned l;
     270             :         unsigned* waiting_map;
     271             :         unsigned waiting_mac;
     272             :         char esc_buffer[ESC_MAX];
     273             :         bit_vect_t* block_enabled;
     274             : 
     275             :         /* maps the disks to handles */
     276          11 :         handle = handle_mapping(state, &diskmax);
     277             : 
     278             :         /* rehash buffers */
     279          11 :         rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
     280             : 
     281             :         /* we need 1 * data + 2 * parity */
     282          11 :         buffermax = diskmax + 2 * state->level;
     283             : 
     284             :         /* initialize the io threads */
     285          11 :         io_init(&io, state, state->opt.io_cache, buffermax, scrub_data_reader, handle, diskmax, scrub_parity_reader, 0, parity_handle, state->level);
     286             : 
     287             :         /* possibly waiting disks */
     288          11 :         waiting_mac = diskmax > RAID_PARITY_MAX ? diskmax : RAID_PARITY_MAX;
     289          11 :         waiting_map = malloc_nofail(waiting_mac * sizeof(unsigned));
     290             : 
     291          11 :         soft_error = 0;
     292          11 :         silent_error = 0;
     293          11 :         io_error = 0;
     294             : 
     295          11 :         msg_progress("Selecting...\n");
     296             : 
     297             :         /* first count the number of blocks to process */
     298          11 :         countmax = 0;
     299          11 :         countlast = 0;
     300          11 :         block_enabled = calloc_nofail(1, bit_vect_size(blockmax)); /* preinitialize to 0 */
     301       56156 :         for (blockcur = blockstart; blockcur < blockmax; ++blockcur) {
     302       56145 :                 if (!block_is_enabled(plan, &countlast, blockcur))
     303       30348 :                         continue;
     304       25797 :                 bit_vect_set(block_enabled, blockcur);
     305       25797 :                 ++countmax;
     306             :         }
     307             : 
     308             :         /* compute the autosave size for all disk, even if not read */
     309             :         /* this makes sense because the speed should be almost the same */
     310             :         /* if the disks are read in parallel */
     311          11 :         autosavelimit = state->autosave / (diskmax * state->block_size);
     312          11 :         autosavemissing = countmax; /* blocks to do */
     313          11 :         autosavedone = 0; /* blocks done */
     314             : 
     315             :         /* drop until now */
     316          11 :         state_usage_waste(state);
     317             : 
     318          11 :         countsize = 0;
     319          11 :         countpos = 0;
     320             : 
     321          11 :         msg_progress("Scrubbing...\n");
     322             : 
     323             :         /* start all the worker threads */
     324          11 :         io_start(&io, blockstart, blockmax, block_enabled);
     325             : 
     326          11 :         int alert = state_progress_begin(state, blockstart, blockmax, countmax);
     327          11 :         if (alert > 0)
     328           0 :                 goto end;
     329          11 :         if (alert < 0)
     330           0 :                 goto bail;
     331             : 
     332       25797 :         while (1) {
     333             :                 unsigned char* buffer_recov[LEV_MAX];
     334             :                 snapraid_info info;
     335             :                 int error_on_this_block;
     336             :                 int silent_error_on_this_block;
     337             :                 int io_error_on_this_block;
     338             :                 int block_is_unsynced;
     339             :                 int rehash;
     340             :                 void** buffer;
     341             : 
     342             :                 /* go to the next block */
     343       25808 :                 blockcur = io_read_next(&io, &buffer);
     344       25808 :                 if (blockcur >= blockmax)
     345          11 :                         break;
     346             : 
     347             :                 /* until now is scheduling */
     348       25797 :                 state_usage_sched(state);
     349             : 
     350             :                 /* one more block processed for autosave */
     351       25797 :                 ++autosavedone;
     352       25797 :                 --autosavemissing;
     353             : 
     354             :                 /* by default process the block, and skip it if something goes wrong */
     355       25797 :                 error_on_this_block = 0;
     356       25797 :                 silent_error_on_this_block = 0;
     357       25797 :                 io_error_on_this_block = 0;
     358             : 
     359             :                 /* if all the blocks at this address are synced */
     360             :                 /* if not, parity is not even checked */
     361       25797 :                 block_is_unsynced = 0;
     362             : 
     363             :                 /* get block specific info */
     364       25797 :                 info = info_get(&state->infoarr, blockcur);
     365             : 
     366             :                 /* if we have to use the old hash */
     367       25797 :                 rehash = info_get_rehash(info);
     368             : 
     369             :                 /* for each disk, process the block */
     370      180579 :                 for (j = 0; j < diskmax; ++j) {
     371             :                         struct snapraid_task* task;
     372             :                         int read_size;
     373             :                         unsigned char hash[HASH_MAX];
     374             :                         struct snapraid_block* block;
     375             :                         int file_is_unsynced;
     376             :                         struct snapraid_disk* disk;
     377             :                         struct snapraid_file* file;
     378             :                         block_off_t file_pos;
     379             :                         unsigned diskcur;
     380             : 
     381             :                         /* if the file on this disk is synced */
     382             :                         /* if not, silent errors are assumed as expected error */
     383      154782 :                         file_is_unsynced = 0;
     384             : 
     385             :                         /* until now is misc */
     386      154782 :                         state_usage_misc(state);
     387             : 
     388             :                         /* get the next task */
     389      154782 :                         task = io_data_read(&io, &diskcur, waiting_map, &waiting_mac);
     390             : 
     391             :                         /* until now is disk */
     392      154782 :                         state_usage_disk(state, handle, waiting_map, waiting_mac);
     393             : 
     394             :                         /* get the task results */
     395      154782 :                         disk = task->disk;
     396      154782 :                         block = task->block;
     397      154782 :                         file = task->file;
     398      154782 :                         file_pos = task->file_pos;
     399      154782 :                         read_size = task->read_size;
     400             : 
     401             :                         /* by default no rehash in case of "continue" */
     402      154782 :                         rehandle[diskcur].block = 0;
     403             : 
     404             :                         /* if the disk position is not used */
     405      154782 :                         if (!disk)
     406        2729 :                                 continue;
     407             : 
     408      154782 :                         state_usage_file(state, disk, file);
     409             : 
     410             :                         /* if the block is unsynced, errors are expected */
     411      154782 :                         if (block_has_invalid_parity(block)) {
     412             :                                 /* report that the block and the file are not synced */
     413           0 :                                 block_is_unsynced = 1;
     414           0 :                                 file_is_unsynced = 1;
     415             :                                 /* follow */
     416             :                         }
     417             : 
     418             :                         /* if the block is not used */
     419      154782 :                         if (!block_has_file(block))
     420        1809 :                                 continue;
     421             : 
     422             :                         /* if the block is unsynced, errors are expected */
     423      152973 :                         if (task->is_timestamp_different) {
     424             :                                 /* report that the block and the file are not synced */
     425           0 :                                 block_is_unsynced = 1;
     426           0 :                                 file_is_unsynced = 1;
     427             :                                 /* follow */
     428             :                         }
     429             : 
     430             :                         /* handle error conditions */
     431      152973 :                         if (task->state == TASK_STATE_IOERROR) {
     432             :                                 /* LCOV_EXCL_START */
     433             :                                 ++io_error;
     434             :                                 goto bail;
     435             :                                 /* LCOV_EXCL_STOP */
     436             :                         }
     437      152973 :                         if (task->state == TASK_STATE_ERROR) {
     438             :                                 /* LCOV_EXCL_START */
     439             :                                 ++soft_error;
     440             :                                 goto bail;
     441             :                                 /* LCOV_EXCL_STOP */
     442             :                         }
     443      152973 :                         if (task->state == TASK_STATE_ERROR_CONTINUE) {
     444           0 :                                 ++soft_error;
     445           0 :                                 error_on_this_block = 1;
     446           0 :                                 continue;
     447             :                         }
     448      152973 :                         if (task->state == TASK_STATE_IOERROR_CONTINUE) {
     449           0 :                                 ++io_error;
     450           0 :                                 if (io_error >= state->opt.io_error_limit) {
     451             :                                         /* LCOV_EXCL_START */
     452             :                                         log_fatal(EIO, "DANGER! Too many input/output errors in the %s disk. It isn't possible to continue.\n", disk->dir);
     453             :                                         log_fatal(EIO, "Stopping at block %u\n", blockcur);
     454             :                                         goto bail;
     455             :                                         /* LCOV_EXCL_STOP */
     456             :                                 }
     457             : 
     458             :                                 /* otherwise continue */
     459           0 :                                 io_error_on_this_block = 1;
     460           0 :                                 continue;
     461             :                         }
     462      152973 :                         if (task->state != TASK_STATE_DONE) {
     463             :                                 /* LCOV_EXCL_START */
     464             :                                 log_fatal(EINTERNAL, "Internal inconsistency in task state\n");
     465             :                                 os_abort();
     466             :                                 /* LCOV_EXCL_STOP */
     467             :                         }
     468             : 
     469      152973 :                         countsize += read_size;
     470             : 
     471             :                         /* now compute the hash */
     472      152973 :                         if (rehash) {
     473       27203 :                                 memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
     474             : 
     475             :                                 /* compute the new hash, and store it */
     476       27203 :                                 rehandle[diskcur].block = block;
     477       27203 :                                 memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
     478             :                         } else {
     479      125770 :                                 memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
     480             :                         }
     481             : 
     482             :                         /* until now is hash */
     483      152973 :                         state_usage_hash(state);
     484             : 
     485      152973 :                         if (block_has_updated_hash(block)) {
     486             :                                 /* compare the hash */
     487      152973 :                                 if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) {
     488         920 :                                         unsigned diff = memdiff(hash, block->hash, BLOCK_HASH_SIZE);
     489             : 
     490             :                                         /* it's a silent error only if we are dealing with synced files */
     491         920 :                                         if (file_is_unsynced) {
     492           0 :                                                 log_tag("error:%u:%s:%s: Data error at position %u, diff hash bits %u/%u\n", blockcur, disk->name, esc_tag(file->sub, esc_buffer), file_pos, diff, BLOCK_HASH_SIZE * 8);
     493           0 :                                                 ++soft_error;
     494           0 :                                                 error_on_this_block = 1;
     495             :                                         } else {
     496         920 :                                                 log_tag("error_data:%u:%s:%s: Data error at position %u, diff hash bits %u/%u\n", blockcur, disk->name, esc_tag(file->sub, esc_buffer), file_pos, diff, BLOCK_HASH_SIZE * 8);
     497         920 :                                                 log_error(EDATA, "Data error in file '%s' at position '%u', diff hash bits %u/%u\n", task->path, file_pos, diff, BLOCK_HASH_SIZE * 8);
     498         920 :                                                 ++silent_error;
     499         920 :                                                 silent_error_on_this_block = 1;
     500             :                                         }
     501         920 :                                         continue;
     502             :                                 }
     503             :                         }
     504             :                 }
     505             : 
     506             :                 /* buffers for parity read and not computed */
     507      180579 :                 for (l = 0; l < state->level; ++l)
     508      154782 :                         buffer_recov[l] = buffer[diskmax + state->level + l];
     509       25797 :                 for (; l < LEV_MAX; ++l)
     510           0 :                         buffer_recov[l] = 0;
     511             : 
     512             :                 /* until now is misc */
     513       25797 :                 state_usage_misc(state);
     514             : 
     515             :                 /* read the parity */
     516      180579 :                 for (l = 0; l < state->level; ++l) {
     517             :                         struct snapraid_task* task;
     518             :                         unsigned levcur;
     519             : 
     520      154782 :                         task = io_parity_read(&io, &levcur, waiting_map, &waiting_mac);
     521             : 
     522             :                         /* until now is parity */
     523      154782 :                         state_usage_parity(state, waiting_map, waiting_mac);
     524             : 
     525             :                         /* handle error conditions */
     526      154782 :                         if (task->state == TASK_STATE_IOERROR) {
     527             :                                 /* LCOV_EXCL_START */
     528             :                                 ++io_error;
     529             :                                 goto bail;
     530             :                                 /* LCOV_EXCL_STOP */
     531             :                         }
     532      154782 :                         if (task->state == TASK_STATE_ERROR) {
     533             :                                 /* LCOV_EXCL_START */
     534             :                                 ++soft_error;
     535             :                                 goto bail;
     536             :                                 /* LCOV_EXCL_STOP */
     537             :                         }
     538      154782 :                         if (task->state == TASK_STATE_ERROR_CONTINUE) {
     539           0 :                                 ++soft_error;
     540           0 :                                 error_on_this_block = 1;
     541             : 
     542             :                                 /* if continuing on error, clear the missing buffer */
     543           0 :                                 buffer_recov[levcur] = 0;
     544           0 :                                 continue;
     545             :                         }
     546      154782 :                         if (task->state == TASK_STATE_IOERROR_CONTINUE) {
     547           0 :                                 ++io_error;
     548           0 :                                 if (io_error >= state->opt.io_error_limit) {
     549             :                                         /* LCOV_EXCL_START */
     550             :                                         log_fatal(EIO, "DANGER! Too many input/output errors in the %s disk. It isn't possible to continue.\n", lev_name(levcur));
     551             :                                         log_fatal(EIO, "Stopping at block %u\n", blockcur);
     552             :                                         goto bail;
     553             :                                         /* LCOV_EXCL_STOP */
     554             :                                 }
     555             : 
     556             :                                 /* otherwise continue */
     557           0 :                                 io_error_on_this_block = 1;
     558             : 
     559             :                                 /* if continuing on error, clear the missing buffer */
     560           0 :                                 buffer_recov[levcur] = 0;
     561           0 :                                 continue;
     562             :                         }
     563      154782 :                         if (task->state != TASK_STATE_DONE) {
     564             :                                 /* LCOV_EXCL_START */
     565             :                                 log_fatal(EINTERNAL, "Internal inconsistency in task state\n");
     566             :                                 os_abort();
     567             :                                 /* LCOV_EXCL_STOP */
     568             :                         }
     569             :                 }
     570             : 
     571             :                 /* if we have read all the data required and it's correct, proceed with the parity check */
     572       25797 :                 if (!error_on_this_block && !silent_error_on_this_block && !io_error_on_this_block) {
     573             : 
     574             :                         /* compute the parity */
     575       24877 :                         raid_gen(diskmax, state->level, state->block_size, buffer);
     576             : 
     577             :                         /* compare the parity */
     578      174139 :                         for (l = 0; l < state->level; ++l) {
     579      149262 :                                 if (buffer_recov[l] && memcmp(buffer[diskmax + l], buffer_recov[l], state->block_size) != 0) {
     580           0 :                                         unsigned diff = memdiff(buffer[diskmax + l], buffer_recov[l], state->block_size);
     581             : 
     582             :                                         /* it's a silent error only if we are dealing with synced blocks */
     583           0 :                                         if (block_is_unsynced) {
     584           0 :                                                 log_tag("parity_error:%u:%s: Data error, diff parity bits %u/%u\n", blockcur, lev_config_name(l), diff, state->block_size * 8);
     585           0 :                                                 ++soft_error;
     586           0 :                                                 error_on_this_block = 1;
     587             :                                         } else {
     588           0 :                                                 log_tag("parity_error_data:%u:%s: Data error, diff parity bits %u/%u\n", blockcur, lev_config_name(l), diff, state->block_size * 8);
     589           0 :                                                 log_error(EDATA, "Data error in parity '%s' at position '%u', diff parity bits %u/%u\n", lev_config_name(l), blockcur, diff, state->block_size * 8);
     590           0 :                                                 ++silent_error;
     591           0 :                                                 silent_error_on_this_block = 1;
     592             :                                         }
     593             :                                 }
     594             :                         }
     595             : 
     596             :                         /* until now is raid */
     597       24877 :                         state_usage_raid(state);
     598             :                 }
     599             : 
     600       25797 :                 if (silent_error_on_this_block || io_error_on_this_block) {
     601             :                         /* set the error status keeping other info */
     602         920 :                         info_set(&state->infoarr, blockcur, info_set_bad(info));
     603       24877 :                 } else if (error_on_this_block) {
     604             :                         /* do nothing, as this is a generic error */
     605             :                         /* likely caused by a not synced array */
     606             :                 } else {
     607             :                         /* if rehash is needed */
     608       24877 :                         if (rehash) {
     609             :                                 /* store all the new hash already computed */
     610       32466 :                                 for (j = 0; j < diskmax; ++j) {
     611       27828 :                                         if (rehandle[j].block)
     612       27203 :                                                 memcpy(rehandle[j].block->hash, rehandle[j].hash, BLOCK_HASH_SIZE);
     613             :                                 }
     614             :                         }
     615             : 
     616             :                         /* update the time info of the block */
     617             :                         /* and clear any other flag */
     618       24877 :                         info_set(&state->infoarr, blockcur, info_make(now, 0, 0, 0));
     619             :                 }
     620             : 
     621             :                 /* mark the state as needing write */
     622       25797 :                 state->need_write = 1;
     623             : 
     624             :                 /* count the number of processed block */
     625       25797 :                 ++countpos;
     626             : 
     627             :                 /* progress */
     628       25797 :                 if (state_progress(state, &io, blockcur, countpos, countmax, countsize)) {
     629             :                         /* LCOV_EXCL_START */
     630             :                         break;
     631             :                         /* LCOV_EXCL_STOP */
     632             :                 }
     633             : 
     634             :                 /* thermal control */
     635       25797 :                 if (state_thermal_alarm(state)) {
     636             :                         /* until now is misc */
     637           0 :                         state_usage_misc(state);
     638             : 
     639           0 :                         state_progress_stop(state);
     640             : 
     641           0 :                         state_thermal_cooldown(state);
     642             : 
     643           0 :                         state_progress_restart(state);
     644             : 
     645             :                         /* drop until now */
     646           0 :                         state_usage_waste(state);
     647             :                 }
     648             : 
     649             :                 /* autosave */
     650       25797 :                 if (state->autosave != 0
     651           0 :                         && autosavedone >= autosavelimit /* if we have reached the limit */
     652           0 :                         && autosavemissing >= autosavelimit /* if we have at least a full step to do */
     653             :                 ) {
     654           0 :                         autosavedone = 0; /* restart the counter */
     655             : 
     656             :                         /* until now is misc */
     657           0 :                         state_usage_misc(state);
     658             : 
     659           0 :                         state_progress_stop(state);
     660             : 
     661           0 :                         msg_progress("Autosaving...\n");
     662           0 :                         state_write(state);
     663             : 
     664           0 :                         state_progress_restart(state);
     665             : 
     666             :                         /* drop until now */
     667           0 :                         state_usage_waste(state);
     668             :                 }
     669             :         }
     670             : 
     671          11 : end:
     672          11 :         state_progress_end(state, countpos, countmax, countsize, "Nothing to scrub. Use the -p PLAN option to select a different plan, like -p full.\n");
     673             : 
     674             :         /* save the new state if required */
     675          11 :         if (state->need_write || state->opt.force_content_write)
     676           9 :                 state_write(state);
     677             : 
     678          11 :         state_usage_print(state);
     679             : 
     680          11 :         if (soft_error || silent_error || io_error) {
     681           1 :                 msg_status("\n");
     682           1 :                 msg_status("%8u soft errors\n", soft_error);
     683           1 :                 msg_status("%8u io errors\n", io_error);
     684           1 :                 msg_status("%8u data errors\n", silent_error);
     685             :         } else {
     686          10 :                 msg_status("Everything OK\n");
     687             :         }
     688             : 
     689          11 :         if (soft_error)
     690           0 :                 log_fatal(ESOFT, "WARNING! Unexpected soft errors!\n");
     691          11 :         if (io_error)
     692           0 :                 log_fatal(EIO, "DANGER! Unexpected input/output errors! The failing blocks are now marked as bad!\n");
     693          11 :         if (silent_error)
     694           1 :                 log_fatal(EDATA, "DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n");
     695          11 :         if (io_error || silent_error) {
     696           1 :                 log_fatal(ESOFT, "Use 'snapraid status' to list the bad blocks.\n");
     697           1 :                 log_fatal(ESOFT, "Use 'snapraid -e fix' to recover them.\n");
     698           1 :                 log_fatal(ESOFT, "Use 'snapraid -p bad scrub' to recheck after fixing to clear the bad state.\n");
     699             :         }
     700             : 
     701          11 :         log_tag("summary:error_soft:%u\n", soft_error);
     702          11 :         log_tag("summary:error_io:%u\n", io_error);
     703          11 :         log_tag("summary:error_data:%u\n", silent_error);
     704          11 :         if (soft_error + silent_error + io_error == 0)
     705          10 :                 log_tag("summary:exit:ok\n");
     706           1 :         else if (silent_error + io_error == 0)
     707           0 :                 log_tag("summary:exit:warning\n");
     708             :         else
     709           1 :                 log_tag("summary:exit:error\n");
     710          11 :         log_flush();
     711             : 
     712          11 : bail:
     713             :         /* stop all the worker threads */
     714          11 :         io_stop(&io);
     715             : 
     716          77 :         for (j = 0; j < diskmax; ++j) {
     717          66 :                 struct snapraid_file* file = handle[j].file;
     718          66 :                 struct snapraid_disk* disk = handle[j].disk;
     719          66 :                 ret = handle_close(&handle[j]);
     720          66 :                 if (ret == -1) {
     721             :                         /* LCOV_EXCL_START */
     722             :                         log_tag("%s:%u:%s:%s: Close error. %s.\n", es(errno), blockcur, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
     723             :                         log_fatal_errno(errno, disk->name);
     724             : 
     725             :                         if (is_hw(errno)) {
     726             :                                 ++io_error;
     727             :                         } else {
     728             :                                 ++soft_error;
     729             :                         }
     730             :                         /* continue, as we are already exiting */
     731             :                         /* LCOV_EXCL_STOP */
     732             :                 }
     733             :         }
     734             : 
     735          11 :         free(handle);
     736          11 :         free(rehandle_alloc);
     737          11 :         free(waiting_map);
     738          11 :         io_done(&io);
     739          11 :         free(block_enabled);
     740             : 
     741          11 :         if (state->opt.expect_recoverable) {
     742           1 :                 if (soft_error + silent_error + io_error == 0)
     743           0 :                         return -1;
     744             :         } else {
     745          10 :                 if (soft_error + silent_error + io_error != 0)
     746           0 :                         return -1;
     747             :         }
     748             : 
     749          11 :         if (alert < 0)
     750           0 :                 return -1;
     751             : 
     752          11 :         return 0;
     753             : }
     754             : 
     755             : /**
     756             :  * Return a * b / c approximated to the upper value.
     757             :  */
     758           2 : static uint32_t md(uint32_t a, uint32_t b, uint32_t c)
     759             : {
     760           2 :         uint64_t v = a;
     761             : 
     762           2 :         v *= b;
     763           2 :         v += c - 1;
     764           2 :         v /= c;
     765             : 
     766           2 :         return v;
     767             : }
     768             : 
     769          11 : int state_scrub(struct snapraid_state* state, int plan100, int olderthan)
     770             : {
     771             :         block_off_t blockmax;
     772             :         block_off_t countlimit;
     773             :         block_off_t count;
     774             :         time_t recentlimit;
     775             :         int ret;
     776             :         struct snapraid_parity_handle parity_handle[LEV_MAX];
     777             :         struct snapraid_plan ps;
     778             :         unsigned process_error;
     779             :         time_t now;
     780             :         unsigned l;
     781             : 
     782             :         /* get the present time */
     783          11 :         now = time(0);
     784             : 
     785          11 :         msg_progress("Initializing...\n");
     786             : 
     787          11 :         if ((plan100 == SCRUB_BAD || plan100 == SCRUB_NEW || plan100 == SCRUB_FULL)
     788           4 :                 && olderthan >= 0) {
     789             :                 /* LCOV_EXCL_START */
     790             :                 log_fatal(EUSER, "You can specify -o, --older-than only with a numeric percentage.\n");
     791             :                 exit(EXIT_FAILURE);
     792             :                 /* LCOV_EXCL_STOP */
     793             :         }
     794             : 
     795          11 :         blockmax = parity_allocated_size(state);
     796             : 
     797             :         /* preinitialize to avoid warnings */
     798          11 :         countlimit = 0;
     799          11 :         recentlimit = 0;
     800             : 
     801          11 :         ps.state = state;
     802          11 :         if (state->opt.force_scrub_even) {
     803           1 :                 ps.plan = SCRUB_EVEN;
     804          10 :         } else if (plan100 == SCRUB_FULL) {
     805           2 :                 ps.plan = SCRUB_FULL;
     806           2 :                 msg_progress("Scrub plan: full. All data blocks will be checked.\n");
     807           8 :         } else if (plan100 == SCRUB_NEW) {
     808           1 :                 ps.plan = SCRUB_NEW;
     809           1 :                 msg_progress("Scrub plan: new. Only blocks that have never been scrubbed will be checked.\n");
     810           7 :         } else if (plan100 == SCRUB_BAD) {
     811           1 :                 ps.plan = SCRUB_BAD;
     812           1 :                 msg_progress("Scrub plan: bad. Only blocks previously marked as bad will be checked.\n");
     813           6 :         } else if (state->opt.force_scrub_at) {
     814             :                 /* scrub the specified amount of blocks */
     815           4 :                 ps.plan = SCRUB_AUTO;
     816           4 :                 countlimit = state->opt.force_scrub_at;
     817           4 :                 recentlimit = now;
     818             :         } else {
     819           2 :                 ps.plan = SCRUB_AUTO;
     820           2 :                 if (plan100 >= 0) {
     821           1 :                         countlimit = md(blockmax, plan100, 10000);
     822             :                 } else {
     823             :                         /* by default scrub 8.33% of the array (100/12=8.(3)) */
     824           1 :                         countlimit = md(blockmax, 1, 12);
     825             :                 }
     826             : 
     827           2 :                 if (olderthan >= 0) {
     828           1 :                         recentlimit = now - olderthan * 24 * 3600;
     829             :                 } else {
     830             :                         /* by default use a 10 day time limit */
     831           1 :                         recentlimit = now - 10 * 24 * 3600;
     832             :                 }
     833             : 
     834           2 :                 if (plan100 >= 0) {
     835           1 :                         if (olderthan >= 0)
     836           0 :                                 msg_progress("Scrub plan: auto. %.1f%% of the array, older than %d days, will be checked.\n", plan100 / 100.0, olderthan);
     837             :                         else
     838           1 :                                 msg_progress("Scrub plan: auto. %.1f%% of the array, older than 10 days, will be checked.\n", plan100 / 100.0);
     839             :                 } else {
     840           1 :                         if (olderthan >= 0)
     841           1 :                                 msg_progress("Scrub plan: auto. 8.3%% of the array, older than %d days, will be checked.\n", olderthan);
     842             :                         else
     843           0 :                                 msg_progress("Scrub plan: auto. 8.3%% of the array, older than 10 days, will be checked.\n");
     844             :                 }
     845             :         }
     846             : 
     847          11 :         count = 0;
     848          32 :         for (tommy_node* j = tommy_list_head(&state->bucketlist); j != 0; j = j->next) {
     849          21 :                 struct snapraid_bucket* bucket = j->data;
     850          21 :                 count += bucket->count_scrubbed + bucket->count_justsynced;
     851             :         }
     852             : 
     853          11 :         if (!count) {
     854             :                 /* LCOV_EXCL_START */
     855             :                 log_fatal(EUSER, "The array is empty.\n");
     856             :                 exit(EXIT_FAILURE);
     857             :                 /* LCOV_EXCL_STOP */
     858             :         }
     859             : 
     860             :         /* compute the limits from count/recentlimit */
     861          11 :         if (ps.plan == SCRUB_AUTO) {
     862             :                 /* no more than the full count */
     863           6 :                 if (countlimit > count)
     864           2 :                         countlimit = count;
     865             : 
     866             :                 /* by default process everything */
     867           6 :                 ps.timelimit = now;
     868           6 :                 ps.lastlimit = 0;
     869             : 
     870           6 :                 tommy_node* j = tommy_list_head(&state->bucketlist);
     871           6 :                 block_off_t processed_count = 0;
     872          11 :                 while (j) {
     873           9 :                         struct snapraid_bucket* bucket = j->data;
     874           9 :                         block_off_t bucket_count = bucket->count_justsynced + bucket->count_scrubbed;
     875             : 
     876           9 :                         if (bucket->time_at > recentlimit) {
     877           1 :                                 ps.timelimit = recentlimit;
     878           1 :                                 ps.lastlimit = 0;
     879           1 :                                 break;
     880             :                         }
     881             : 
     882           8 :                         if (processed_count + bucket_count > countlimit) {
     883           3 :                                 ps.timelimit = bucket->time_at;
     884           3 :                                 ps.lastlimit = countlimit - processed_count;
     885           3 :                                 processed_count = countlimit;
     886           3 :                                 break;
     887             :                         }
     888             : 
     889           5 :                         processed_count += bucket_count;
     890           5 :                         j = j->next;
     891             :                 }
     892             : 
     893             :                 /* if nothing to scrub, disable also other limits */
     894           6 :                 if (processed_count == 0) {
     895           1 :                         ps.timelimit = 0;
     896           1 :                         ps.lastlimit = 0;
     897             :                 }
     898             : 
     899           6 :                 log_tag("count_limit:%u\n", countlimit);
     900           6 :                 log_tag("time_limit:%" PRIu64 "\n", (uint64_t)ps.timelimit);
     901           6 :                 log_tag("last_limit:%u\n", ps.lastlimit);
     902             :         } else {
     903             :                 /* avoid compiler warnings */
     904           5 :                 ps.timelimit = 0;
     905           5 :                 ps.lastlimit = 0;
     906             :         }
     907             : 
     908             :         /* open the file for reading */
     909          77 :         for (l = 0; l < state->level; ++l) {
     910          66 :                 ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
     911          66 :                 if (ret == -1) {
     912             :                         /* LCOV_EXCL_START */
     913             :                         log_tag("parity_%s:%u:%s: Open error. %s.\n", es(errno), blockmax, lev_config_name(l), strerror(errno));
     914             :                         log_fatal_errno(errno, lev_config_name(l));
     915             :                         exit(EXIT_FAILURE);
     916             :                         /* LCOV_EXCL_STOP */
     917             :                 }
     918             :         }
     919             : 
     920          11 :         process_error = 0;
     921             : 
     922          11 :         ret = state_scrub_process(state, parity_handle, 0, blockmax, &ps, now);
     923          11 :         if (ret == -1) {
     924           0 :                 ++process_error;
     925             :                 /* continue, as we are already exiting */
     926             :         }
     927             : 
     928          77 :         for (l = 0; l < state->level; ++l) {
     929          66 :                 ret = parity_close(&parity_handle[l]);
     930          66 :                 if (ret == -1) {
     931             :                         /* LCOV_EXCL_START */
     932             :                         log_tag("parity_%s:%u:%s: Close error. %s.\n", es(errno), blockmax, lev_config_name(l), strerror(errno));
     933             :                         log_fatal_errno(errno, lev_config_name(l));
     934             : 
     935             :                         ++process_error;
     936             :                         /* continue, as we are already exiting */
     937             :                         /* LCOV_EXCL_STOP */
     938             :                 }
     939             :         }
     940             : 
     941          11 :         if (process_error != 0)
     942           0 :                 return -1;
     943          11 :         return 0;
     944             : }
     945             : 

Generated by: LCOV version 1.0