LCOV - code coverage report
Current view: top level - cmdline - sync.c (source / functions) Hit Total Coverage
Test: lcov.info Lines: 426 453 94.0 %
Date: 2015-05-12 Functions: 4 4 100.0 %

          Line data    Source code
       1             : /*
       2             :  * Copyright (C) 2011 Andrea Mazzoleni
       3             :  *
       4             :  * This program is free software: you can redistribute it and/or modify
       5             :  * it under the terms of the GNU General Public License as published by
       6             :  * the Free Software Foundation, either version 3 of the License, or
       7             :  * (at your option) any later version.
       8             :  *
       9             :  * This program is distributed in the hope that it will be useful,
      10             :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      12             :  * GNU General Public License for more details.
      13             :  *
      14             :  * You should have received a copy of the GNU General Public License
      15             :  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
      16             :  */
      17             : 
      18             : #include "portable.h"
      19             : 
      20             : #include "support.h"
      21             : #include "elem.h"
      22             : #include "state.h"
      23             : #include "parity.h"
      24             : #include "handle.h"
      25             : #include "raid/raid.h"
      26             : 
      27             : /****************************************************************************/
      28             : /* hash */
      29             : 
      30           6 : static int state_hash_process(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, int* skip_sync)
      31             : {
      32             :         struct snapraid_handle* handle;
      33             :         unsigned diskmax;
      34             :         block_off_t i;
      35             :         unsigned j;
      36             :         void* buffer;
      37             :         data_off_t countsize;
      38             :         block_off_t countpos;
      39             :         block_off_t countmax;
      40             :         int ret;
      41             :         unsigned error;
      42             :         unsigned io_error;
      43             : 
      44             :         /* maps the disks to handles */
      45           6 :         handle = handle_map(state, &diskmax);
      46             : 
      47             :         /* buffer for reading */
      48           6 :         buffer = malloc_nofail(state->block_size);
      49           6 :         if (!state->opt.skip_self)
      50           0 :                 mtest_vector(1, state->block_size, &buffer);
      51             : 
      52           6 :         error = 0;
      53           6 :         io_error = 0;
      54             : 
      55             :         /* first count the number of blocks to process */
      56           6 :         countmax = 0;
      57          42 :         for (j = 0; j < diskmax; ++j) {
      58          36 :                 struct snapraid_disk* disk = handle[j].disk;
      59             : 
      60             :                 /* if no disk, nothing to check */
      61          36 :                 if (!disk)
      62           0 :                         continue;
      63             : 
      64      221700 :                 for (i = blockstart; i < blockmax; ++i) {
      65             :                         struct snapraid_block* block;
      66             : 
      67      221664 :                         block = disk_block_get(disk, i);
      68             : 
      69             :                         /* if no file in this block, skip it */
      70      221664 :                         if (!block_has_file(block))
      71       15168 :                                 continue;
      72             : 
      73             :                         /* if the block already has a hash, skip it */
      74      206496 :                         if (block_has_updated_hash(block))
      75      193805 :                                 continue;
      76             : 
      77       12691 :                         ++countmax;
      78             :                 }
      79             :         }
      80             : 
      81             :         /* drop until now */
      82           6 :         state_usage_waste(state);
      83             : 
      84           6 :         countsize = 0;
      85           6 :         countpos = 0;
      86           6 :         if (!state_progress_begin(state, blockstart, blockmax, countmax))
      87           0 :                 goto end;
      88             : 
      89          42 :         for (j = 0; j < diskmax; ++j) {
      90          36 :                 struct snapraid_disk* disk = handle[j].disk;
      91             : 
      92             :                 /* if no disk, nothing to check */
      93          36 :                 if (!disk)
      94           0 :                         continue;
      95             : 
      96      221700 :                 for (i = blockstart; i < blockmax; ++i) {
      97             :                         snapraid_info info;
      98             :                         int rehash;
      99             :                         struct snapraid_block* block;
     100             :                         int read_size;
     101             :                         unsigned char hash[HASH_SIZE];
     102             : 
     103      221664 :                         block = disk_block_get(disk, i);
     104             : 
     105             :                         /* if no file in this block, skip it */
     106      221664 :                         if (!block_has_file(block))
     107      224150 :                                 continue;
     108             : 
     109             :                         /* if the block already has a hash, skip it */
     110      206496 :                         if (block_has_updated_hash(block))
     111      193805 :                                 continue;
     112             : 
     113             :                         /* here only CHG blocks are allowed */
     114       12691 :                         assert(block_state_get(block) == BLOCK_STATE_CHG);
     115             : 
     116             :                         /* get block specific info */
     117       12691 :                         info = info_get(&state->infoarr, i);
     118             : 
     119             :                         /* if we have to use the old hash */
     120       12691 :                         rehash = info_get_rehash(info);
     121             : 
     122             :                         /* until now is CPU */
     123       12691 :                         state_usage_cpu(state);
     124             : 
     125             :                         /* if the file is different than the current one, close it */
     126       12691 :                         if (handle[j].file != 0 && handle[j].file != block_file_get(block)) {
     127             :                                 /* keep a pointer at the file we are going to close for error reporting */
     128        4710 :                                 struct snapraid_file* file = handle[j].file;
     129        4710 :                                 ret = handle_close(&handle[j]);
     130        4710 :                                 if (ret == -1) {
     131             :                                         /* LCOV_EXCL_START */
     132             :                                         /* This one is really an unexpected error, because we are only reading */
     133             :                                         /* and closing a descriptor should never fail */
     134             :                                         if (errno == EIO) {
     135             :                                                 log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     136             :                                                 log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n");
     137             :                                                 log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
     138             :                                                 log_fatal("Stopping at block %u\n", i);
     139             :                                                 ++io_error;
     140             :                                                 goto bail;
     141             :                                         }
     142             : 
     143             :                                         log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     144             :                                         log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n");
     145             :                                         log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
     146             :                                         log_fatal("Stopping at block %u\n", i);
     147             :                                         ++error;
     148             :                                         goto bail;
     149             :                                         /* LCOV_EXCL_STOP */
     150             :                                 }
     151             :                         }
     152             : 
     153       12691 :                         ret = handle_open(&handle[j], block_file_get(block), state->file_mode, log_error, 0);
     154       12691 :                         if (ret == -1) {
     155             :                                 /* file we have tried to open for error reporting */
     156           6 :                                 struct snapraid_file* file = block_file_get(block);
     157           6 :                                 if (errno == EIO) {
     158             :                                         /* LCOV_EXCL_START */
     159             :                                         log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     160             :                                         log_fatal("DANGER! Unexpected input/output open error in a data disk, it isn't possible to sync.\n");
     161             :                                         log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
     162             :                                         log_fatal("Stopping at block %u\n", i);
     163             :                                         ++io_error;
     164             :                                         goto bail;
     165             :                                         /* LCOV_EXCL_STOP */
     166             :                                 }
     167             : 
     168           6 :                                 if (errno == ENOENT) {
     169           2 :                                         log_tag("error:%u:%s:%s: Open ENOENT error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     170           2 :                                         log_error("Missing file '%s'.\n", handle[j].path);
     171           2 :                                         log_error("WARNING! You cannot modify data disk during a sync.\n");
     172           2 :                                         log_error("Rerun the sync command when finished.\n");
     173           2 :                                         ++error;
     174             :                                         /* if the file is missing, it means that it was removed during sync */
     175             :                                         /* this isn't a serious error, so we skip this block, and continue with others */
     176           2 :                                         continue;
     177             :                                 }
     178             : 
     179           4 :                                 if (errno == EACCES) {
     180           4 :                                         log_tag("error:%u:%s:%s: Open EACCES error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     181           4 :                                         log_error("No access at file '%s'.\n", handle[j].path);
     182           4 :                                         log_error("WARNING! Please fix the access permission in the data disk.\n");
     183           4 :                                         log_error("Rerun the sync command when finished.\n");
     184           4 :                                         ++error;
     185             :                                         /* this isn't a serious error, so we skip this block, and continue with others */
     186           4 :                                         continue;
     187             :                                 }
     188             : 
     189             :                                 /* LCOV_EXCL_START */
     190             :                                 log_tag("error:%u:%s:%s: Open error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     191             :                                 log_fatal("WARNING! Unexpected open error in a data disk, it isn't possible to sync.\n");
     192             :                                 log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
     193             :                                 log_fatal("Stopping to allow recovery. Try with 'snapraid check -f %s'\n", file->sub);
     194             :                                 ++error;
     195             :                                 goto bail;
     196             :                                 /* LCOV_EXCL_STOP */
     197             :                         }
     198             : 
     199             :                         /* check if the file is changed */
     200       12685 :                         if (handle[j].st.st_size != block_file_get(block)->size
     201       12684 :                                 || handle[j].st.st_mtime != block_file_get(block)->mtime_sec
     202       12683 :                                 || STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
     203       12683 :                                 || handle[j].st.st_ino != block_file_get(block)->inode
     204             :                         ) {
     205             :                                 /* file we are processing for error reporting */
     206           3 :                                 struct snapraid_file* file = block_file_get(block);
     207           3 :                                 log_tag("error:%u:%s:%s: Unexpected attribute change\n", i, disk->name, esc(file->sub));
     208           3 :                                 if (handle[j].st.st_size != file->size) {
     209           1 :                                         log_error("Unexpected size change at file '%s' from %" PRIu64 " to %" PRIu64 ".\n", handle[j].path, file->size, handle[j].st.st_size);
     210           2 :                                 } else if (handle[j].st.st_mtime != file->mtime_sec
     211           1 :                                         || STAT_NSEC(&handle[j].st) != file->mtime_nsec) {
     212           1 :                                         log_error("Unexpected time change at file '%s' from %" PRIu64 ".%d to %" PRIu64 ".%d.\n", handle[j].path, file->mtime_sec, file->mtime_nsec, (uint64_t)handle[j].st.st_mtime, (uint32_t)STAT_NSEC(&handle[j].st));
     213             :                                 } else {
     214           1 :                                         log_error("Unexpected inode change from %" PRIu64 " to %" PRIu64 " at file '%s'.\n", file->inode, (uint64_t)handle[j].st.st_ino, handle[j].path);
     215             :                                 }
     216           3 :                                 log_error("WARNING! You cannot modify files during a sync.\n");
     217           3 :                                 log_error("Rerun the sync command when finished.\n");
     218           3 :                                 ++error;
     219             :                                 /* if the file is changed, it means that it was modified during sync */
     220             :                                 /* this isn't a serious error, so we skip this block, and continue with others */
     221           3 :                                 continue;
     222             :                         }
     223             : 
     224       12682 :                         read_size = handle_read(&handle[j], block, buffer, state->block_size, log_fatal, 0);
     225       12682 :                         if (read_size == -1) {
     226             :                                 /* LCOV_EXCL_START */
     227             :                                 /* file we are processing for error reporting */
     228             :                                 struct snapraid_file* file = block_file_get(block);
     229             :                                 if (errno == EIO) {
     230             :                                         log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
     231             :                                         log_fatal("DANGER! Unexpected input/output read error in a data disk, it isn't possible to sync.\n");
     232             :                                         log_fatal("Ensure that disk '%s' is sane and that file '%s' can be read.\n", disk->dir, handle[j].path);
     233             :                                         log_fatal("Stopping at block %u\n", i);
     234             :                                         ++io_error;
     235             :                                         goto bail;
     236             :                                 }
     237             : 
     238             :                                 log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
     239             :                                 log_fatal("WARNING! Unexpected read error in a data disk, it isn't possible to sync.\n");
     240             :                                 log_fatal("Ensure that file '%s' can be read.\n", handle[j].path);
     241             :                                 log_fatal("Stopping to allow recovery. Try with 'snapraid check -f %s'\n", file->sub);
     242             :                                 ++error;
     243             :                                 goto bail;
     244             :                                 /* LCOV_EXCL_STOP */
     245             :                         }
     246             : 
     247             :                         /* until now is disk */
     248       12682 :                         state_usage_disk(state, disk);
     249             : 
     250       12682 :                         countsize += read_size;
     251             : 
     252             :                         /* now compute the hash */
     253       12682 :                         if (rehash) {
     254           0 :                                 memhash(state->prevhash, state->prevhashseed, hash, buffer, read_size);
     255             :                         } else {
     256       12682 :                                 memhash(state->hash, state->hashseed, hash, buffer, read_size);
     257             :                         }
     258             : 
     259             :                         /* copy the hash in the block */
     260       12682 :                         memcpy(block->hash, hash, HASH_SIZE);
     261             : 
     262             :                         /* and mark the block as hashed */
     263       12682 :                         block_state_set(block, BLOCK_STATE_REP);
     264             : 
     265             :                         /* mark the state as needing write */
     266       12682 :                         state->need_write = 1;
     267             : 
     268             :                         /* count the number of processed block */
     269       12682 :                         ++countpos;
     270             : 
     271             :                         /* progress */
     272       12682 :                         if (state_progress(state, i, countpos, countmax, countsize)) {
     273             :                                 /* LCOV_EXCL_START */
     274             :                                 *skip_sync = 1; /* avoid to run the next sync */
     275             :                                 break;
     276             :                                 /* LCOV_EXCL_STOP */
     277             :                         }
     278             :                 }
     279             : 
     280             :                 /* close the last file in the disk */
     281          36 :                 if (handle[j].file != 0) {
     282             :                         /* keep a pointer at the file we are going to close for error reporting */
     283           6 :                         struct snapraid_file* file = handle[j].file;
     284           6 :                         ret = handle_close(&handle[j]);
     285           6 :                         if (ret == -1) {
     286             :                                 /* LCOV_EXCL_START */
     287             :                                 /* This one is really an unexpected error, because we are only reading */
     288             :                                 /* and closing a descriptor should never fail */
     289             :                                 if (errno == EIO) {
     290             :                                         log_tag("error:%u:%s:%s: Close EIO error. %s\n", blockmax, disk->name, esc(file->sub), strerror(errno));
     291             :                                         log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n");
     292             :                                         log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
     293             :                                         log_fatal("Stopping at block %u\n", blockmax);
     294             :                                         ++io_error;
     295             :                                         goto bail;
     296             :                                 }
     297             : 
     298             :                                 log_tag("error:%u:%s:%s: Close error. %s\n", blockmax, disk->name, esc(file->sub), strerror(errno));
     299             :                                 log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n");
     300             :                                 log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
     301             :                                 log_fatal("Stopping at block %u\n", blockmax);
     302             :                                 ++error;
     303             :                                 goto bail;
     304             :                                 /* LCOV_EXCL_STOP */
     305             :                         }
     306             :                 }
     307             :         }
     308             : 
     309             : end:
     310           6 :         state_progress_end(state, countpos, countmax, countsize);
     311             : 
     312             :         /* note that at this point no io_error is possible */
     313             :         /* becasue at the first one we bail out */
     314           6 :         assert(io_error == 0);
     315             : 
     316           6 :         if (error) {
     317           5 :                 msg_status("\n");
     318           5 :                 msg_status("%8u file errors\n", error);
     319             :         } else {
     320             :                 /* print the result only if processed something */
     321           1 :                 if (countpos != 0)
     322           1 :                         msg_status("Everything OK\n");
     323             :         }
     324             : 
     325           6 :         if (error)
     326           5 :                 log_fatal("WARNING! Unexpected file errors!\n");
     327             : 
     328           6 :         log_tag("hash_summary:error_file:%u\n", error);
     329             : 
     330             :         /* proceed without bailing out */
     331           6 :         goto finish;
     332             : 
     333             : bail:
     334             :         /* on bail, don't run the next sync */
     335           0 :         *skip_sync = 1;
     336             : 
     337             :         /* close files left open */
     338           0 :         for (j = 0; j < diskmax; ++j) {
     339           0 :                 struct snapraid_file* file = handle[j].file;
     340           0 :                 struct snapraid_disk* disk = handle[j].disk;
     341           0 :                 ret = handle_close(&handle[j]);
     342           0 :                 if (ret == -1) {
     343           0 :                         log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     344           0 :                         log_fatal("DANGER! Unexpected close error in a data disk.\n");
     345           0 :                         ++error;
     346             :                         /* continue, as we are already exiting */
     347             :                 }
     348             :         }
     349             : 
     350             : finish:
     351           6 :         free(handle);
     352           6 :         free(buffer);
     353             : 
     354           6 :         if (error + io_error != 0)
     355           5 :                 return -1;
     356           1 :         return 0;
     357             : }
     358             : 
     359             : /****************************************************************************/
     360             : /* sync */
     361             : 
     362             : /**
     363             :  * A block that failed the hash check, or that was deleted.
     364             :  */
     365             : struct failed_struct {
     366             :         unsigned index; /**< Index of the failed block. */
     367             :         unsigned size; /**< Size of the block. */
     368             : 
     369             :         struct snapraid_block* block; /**< The failed block, or BLOCK_DELETED for a deleted block */
     370             : };
     371             : 
     372             : /**
     373             :  * Buffer for storing the new hashes.
     374             :  */
     375             : struct snapraid_rehash {
     376             :         unsigned char hash[HASH_SIZE];
     377             :         struct snapraid_block* block;
     378             : };
     379             : 
     380             : /**
     381             :  * Check if we have to process the specified block index ::i.
     382             :  */
     383      723482 : static int block_is_enabled(block_off_t i, struct snapraid_handle* handle, unsigned diskmax)
     384             : {
     385             :         unsigned j;
     386             :         int one_invalid;
     387             :         int one_valid;
     388             : 
     389             :         /* for each disk */
     390      723482 :         one_invalid = 0;
     391      723482 :         one_valid = 0;
     392     5064374 :         for (j = 0; j < diskmax; ++j) {
     393             :                 struct snapraid_block* block;
     394             : 
     395             :                 /* if no disk, nothing to check */
     396     4340892 :                 if (!handle[j].disk)
     397        9342 :                         continue;
     398             : 
     399     4331550 :                 block = disk_block_get(handle[j].disk, i);
     400             : 
     401     4331550 :                 if (block_has_file(block))
     402     4124374 :                         one_valid = 1;
     403             : 
     404     4331550 :                 if (block_has_invalid_parity(block))
     405      356420 :                         one_invalid = 1;
     406             :         }
     407             : 
     408             :         /* if none valid or none invalid, we don't need to update */
     409      723482 :         if (!one_invalid || !one_valid)
     410      563606 :                 return 0;
     411             : 
     412      159876 :         return 1;
     413             : }
     414             : 
     415          62 : static int state_sync_process(struct snapraid_state* state, struct snapraid_parity_handle** parity, block_off_t blockstart, block_off_t blockmax)
     416             : {
     417             :         struct snapraid_handle* handle;
     418             :         void* rehandle_alloc;
     419             :         struct snapraid_rehash* rehandle;
     420             :         unsigned diskmax;
     421             :         block_off_t i;
     422             :         unsigned j;
     423             :         void* buffer_alloc;
     424             :         void** buffer;
     425             :         unsigned buffermax;
     426             :         data_off_t countsize;
     427             :         block_off_t countpos;
     428             :         block_off_t countmax;
     429             :         block_off_t autosavedone;
     430             :         block_off_t autosavelimit;
     431             :         block_off_t autosavemissing;
     432             :         int ret;
     433             :         unsigned error;
     434             :         unsigned silent_error;
     435             :         unsigned io_error;
     436             :         time_t now;
     437             :         struct failed_struct* failed;
     438             :         int* failed_map;
     439             :         unsigned l;
     440             : 
     441             :         /* the sync process assumes that all the hashes are correct */
     442             :         /* including the ones from CHG and DELETED blocks */
     443          62 :         assert(state->clear_past_hash != 0);
     444             : 
     445             :         /* get the present time */
     446          62 :         now = time(0);
     447             : 
     448             :         /* maps the disks to handles */
     449          62 :         handle = handle_map(state, &diskmax);
     450             : 
     451             :         /* rehash buffers */
     452          62 :         rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
     453             : 
     454             :         /* we need 2 * data + 1 * parity + 1 * zero */
     455          62 :         buffermax = 2 * diskmax + state->level + 1;
     456             : 
     457          62 :         buffer = malloc_nofail_vector_align(diskmax, buffermax, state->block_size, &buffer_alloc);
     458          62 :         if (!state->opt.skip_self)
     459           0 :                 mtest_vector(buffermax, state->block_size, buffer);
     460             : 
     461             :         /* fill up the zero buffer */
     462          62 :         memset(buffer[buffermax - 1], 0, state->block_size);
     463          62 :         raid_zero(buffer[buffermax - 1]);
     464             : 
     465          62 :         failed = malloc_nofail(diskmax * sizeof(struct failed_struct));
     466          62 :         failed_map = malloc_nofail(diskmax * sizeof(unsigned));
     467             : 
     468          62 :         error = 0;
     469          62 :         silent_error = 0;
     470          62 :         io_error = 0;
     471             : 
     472             :         /* first count the number of blocks to process */
     473          62 :         countmax = 0;
     474      361803 :         for (i = blockstart; i < blockmax; ++i) {
     475      361741 :                 if (!block_is_enabled(i, handle, diskmax))
     476      281803 :                         continue;
     477       79938 :                 ++countmax;
     478             :         }
     479             : 
     480             :         /* compute the autosave size for all disk, even if not read */
     481             :         /* this makes sense because the speed should be almost the same */
     482             :         /* if the disks are read in parallel */
     483          62 :         autosavelimit = state->autosave / (diskmax * state->block_size);
     484          62 :         autosavemissing = countmax; /* blocks to do */
     485          62 :         autosavedone = 0; /* blocks done */
     486             : 
     487             :         /* drop until now */
     488          62 :         state_usage_waste(state);
     489             : 
     490          62 :         countsize = 0;
     491          62 :         countpos = 0;
     492          62 :         if (!state_progress_begin(state, blockstart, blockmax, countmax))
     493           0 :                 goto end;
     494             : 
     495      361803 :         for (i = blockstart; i < blockmax; ++i) {
     496             :                 unsigned failed_count;
     497             :                 int error_on_this_block;
     498             :                 int silent_error_on_this_block;
     499             :                 int io_error_on_this_block;
     500             :                 int fixed_error_on_this_block;
     501             :                 int parity_needs_to_be_updated;
     502             :                 snapraid_info info;
     503             :                 int rehash;
     504             : 
     505      361741 :                 if (!block_is_enabled(i, handle, diskmax))
     506      281803 :                         continue;
     507             : 
     508             :                 /* one more block processed for autosave */
     509       79938 :                 ++autosavedone;
     510       79938 :                 --autosavemissing;
     511             : 
     512             :                 /* by default process the block, and skip it if something go wrong */
     513       79938 :                 error_on_this_block = 0;
     514       79938 :                 silent_error_on_this_block = 0;
     515       79938 :                 io_error_on_this_block = 0;
     516       79938 :                 fixed_error_on_this_block = 0;
     517             : 
     518             :                 /* keep track of the number of failed blocks */
     519       79938 :                 failed_count = 0;
     520             : 
     521             :                 /* get block specific info */
     522       79938 :                 info = info_get(&state->infoarr, i);
     523             : 
     524             :                 /* if we have to use the old hash */
     525       79938 :                 rehash = info_get_rehash(info);
     526             : 
     527             :                 /* it could happens that all the blocks are EMPTY/BLK and CHG but with the hash */
     528             :                 /* still matching because the specific CHG block was not modified. */
     529             :                 /* Note that CHG/DELETED blocks already present in the content file loaded */
     530             :                 /* have the hash cleared (::clear_past_hash flag), and then they won't never match the hash. */
     531             :                 /* We are treating only CHG blocks created at runtime. */
     532             :                 /* In such case, we can avoid to update parity, because it would be the same as before */
     533       79938 :                 parity_needs_to_be_updated = 0;
     534             : 
     535             :                 /* if the block is marked as bad, we force the parity update */
     536             :                 /* because the bad block may be the result of a wrong parity */
     537       79938 :                 if (info_get_bad(info))
     538           0 :                         parity_needs_to_be_updated = 1;
     539             : 
     540             :                 /* for each disk, process the block */
     541      559566 :                 for (j = 0; j < diskmax; ++j) {
     542             :                         int read_size;
     543             :                         unsigned char hash[HASH_SIZE];
     544             :                         struct snapraid_block* block;
     545             :                         unsigned block_state;
     546      479628 :                         struct snapraid_disk* disk = handle[j].disk;
     547             : 
     548             :                         /* by default no rehash in case of "continue" */
     549      479628 :                         rehandle[j].block = 0;
     550             : 
     551             :                         /* if the disk position is not used */
     552      479628 :                         if (!disk) {
     553             :                                 /* use an empty block */
     554           0 :                                 memset(buffer[j], 0, state->block_size);
     555       88530 :                                 continue;
     556             :                         }
     557             : 
     558             :                         /* get the block */
     559      479628 :                         block = disk_block_get(disk, i);
     560             : 
     561             :                         /* get the state of the block */
     562      479628 :                         block_state = block_state_get(block);
     563             : 
     564             :                         /* if the block has invalid parity, */
     565             :                         /* we have to take care of it in case of recover */
     566      479628 :                         if (block_has_invalid_parity(block)) {
     567             :                                 /* store it in the failed set, because */
     568             :                                 /* the parity may be still computed with the previous content */
     569      178210 :                                 failed[failed_count].index = j;
     570      178210 :                                 failed[failed_count].size = state->block_size;
     571      178210 :                                 failed[failed_count].block = block;
     572      178210 :                                 ++failed_count;
     573             : 
     574             :                                 /* if the block has invalid parity, we have to update the parity */
     575             :                                 /* to include this block change */
     576             :                                 /* This also apply to CHG blocks, but we are going to handle */
     577             :                                 /* later this case to do the updates only if really needed */
     578      178210 :                                 if (block_state != BLOCK_STATE_CHG)
     579       58416 :                                         parity_needs_to_be_updated = 1;
     580             : 
     581             :                                 /* note that DELETE blocks are skipped in the next check */
     582             :                                 /* and we have to store them in the failed blocks */
     583             :                                 /* before skipping */
     584             : 
     585             :                                 /* follow */
     586             :                         }
     587             : 
     588             :                         /* if the block has no file, meanining that it's EMPTY or DELETED, */
     589             :                         /* it doesn't partecipate in the new parity computation */
     590      479628 :                         if (!block_has_file(block)) {
     591             :                                 /* use an empty block */
     592       87628 :                                 memset(buffer[j], 0, state->block_size);
     593       87628 :                                 continue;
     594             :                         }
     595             : 
     596             :                         /* until now is CPU */
     597      392000 :                         state_usage_cpu(state);
     598             : 
     599             :                         /* if the file is different than the current one, close it */
     600      392000 :                         if (handle[j].file != 0 && handle[j].file != block_file_get(block)) {
     601             :                                 /* keep a pointer at the file we are going to close for error reporting */
     602      163209 :                                 struct snapraid_file* file = handle[j].file;
     603      163209 :                                 ret = handle_close(&handle[j]);
     604      163209 :                                 if (ret == -1) {
     605             :                                         /* LCOV_EXCL_START */
     606             :                                         /* This one is really an unexpected error, because we are only reading */
     607             :                                         /* and closing a descriptor should never fail */
     608             :                                         if (errno == EIO) {
     609             :                                                 log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     610             :                                                 log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n");
     611             :                                                 log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
     612             :                                                 log_fatal("Stopping at block %u\n", i);
     613             :                                                 ++io_error;
     614             :                                                 goto bail;
     615             :                                         }
     616             : 
     617             :                                         log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     618             :                                         log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n");
     619             :                                         log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
     620             :                                         log_fatal("Stopping at block %u\n", i);
     621             :                                         ++error;
     622             :                                         goto bail;
     623             :                                         /* LCOV_EXCL_STOP */
     624             :                                 }
     625             :                         }
     626             : 
     627      392000 :                         ret = handle_open(&handle[j], block_file_get(block), state->file_mode, log_error, 0);
     628      392000 :                         if (ret == -1) {
     629             :                                 /* file we have tried to open for error reporting */
     630          12 :                                 struct snapraid_file* file = block_file_get(block);
     631          12 :                                 if (errno == EIO) {
     632             :                                         /* LCOV_EXCL_START */
     633             :                                         log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     634             :                                         log_fatal("DANGER! Unexpected input/output open error in a data disk, it isn't possible to sync.\n");
     635             :                                         log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
     636             :                                         log_fatal("Stopping at block %u\n", i);
     637             :                                         ++io_error;
     638             :                                         goto bail;
     639             :                                         /* LCOV_EXCL_STOP */
     640             :                                 }
     641             : 
     642          12 :                                 if (errno == ENOENT) {
     643           4 :                                         log_tag("error:%u:%s:%s: Open ENOENT error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     644           4 :                                         log_error("Missing file '%s'.\n", handle[j].path);
     645           4 :                                         log_error("WARNING! You cannot modify data disk during a sync.\n");
     646           4 :                                         log_error("Rerun the sync command when finished.\n");
     647           4 :                                         ++error;
     648             :                                         /* if the file is missing, it means that it was removed during sync */
     649             :                                         /* this isn't a serious error, so we skip this block, and continue with others */
     650           4 :                                         error_on_this_block = 1;
     651           4 :                                         continue;
     652             :                                 }
     653             : 
     654           8 :                                 if (errno == EACCES) {
     655           8 :                                         log_tag("error:%u:%s:%s: Open EACCES error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     656           8 :                                         log_error("No access at file '%s'.\n", handle[j].path);
     657           8 :                                         log_error("WARNING! Please fix the access permission in the data disk.\n");
     658           8 :                                         log_error("Rerun the sync command when finished.\n");
     659           8 :                                         ++error;
     660             :                                         /* this isn't a serious error, so we skip this block, and continue with others */
     661           8 :                                         error_on_this_block = 1;
     662           8 :                                         continue;
     663             :                                 }
     664             : 
     665             :                                 /* LCOV_EXCL_START */
     666             :                                 log_tag("error:%u:%s:%s: Open error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
     667             :                                 log_fatal("WARNING! Unexpected open error in a data disk, it isn't possible to sync.\n");
     668             :                                 log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
     669             :                                 log_fatal("Stopping to allow recovery. Try with 'snapraid check -f %s'\n", file->sub);
     670             :                                 ++error;
     671             :                                 goto bail;
     672             :                                 /* LCOV_EXCL_STOP */
     673             :                         }
     674             : 
     675             :                         /* check if the file is changed */
     676      391988 :                         if (handle[j].st.st_size != block_file_get(block)->size
     677      391986 :                                 || handle[j].st.st_mtime != block_file_get(block)->mtime_sec
     678      391984 :                                 || STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
     679      391984 :                                 || handle[j].st.st_ino != block_file_get(block)->inode
     680             :                         ) {
     681             :                                 /* file we are processing for error reporting */
     682           6 :                                 struct snapraid_file* file = block_file_get(block);
     683           6 :                                 log_tag("error:%u:%s:%s: Unexpected attribute change\n", i, disk->name, esc(file->sub));
     684           6 :                                 if (handle[j].st.st_size != file->size) {
     685           2 :                                         log_error("Unexpected size change at file '%s' from %" PRIu64 " to %" PRIu64 ".\n", handle[j].path, file->size, handle[j].st.st_size);
     686           4 :                                 } else if (handle[j].st.st_mtime != file->mtime_sec
     687           2 :                                         || STAT_NSEC(&handle[j].st) != file->mtime_nsec) {
     688           2 :                                         log_error("Unexpected time change at file '%s' from %" PRIu64 ".%d to %" PRIu64 ".%d.\n", handle[j].path, file->mtime_sec, file->mtime_nsec, (uint64_t)handle[j].st.st_mtime, (uint32_t)STAT_NSEC(&handle[j].st));
     689             :                                 } else {
     690           2 :                                         log_error("Unexpected inode change from %" PRIu64 " to %" PRIu64 " at file '%s'.\n", file->inode, (uint64_t)handle[j].st.st_ino, handle[j].path);
     691             :                                 }
     692           6 :                                 log_error("WARNING! You cannot modify files during a sync.\n");
     693           6 :                                 log_error("Rerun the sync command when finished.\n");
     694           6 :                                 ++error;
     695             :                                 /* if the file is changed, it means that it was modified during sync */
     696             :                                 /* this isn't a serious error, so we skip this block, and continue with others */
     697           6 :                                 error_on_this_block = 1;
     698           6 :                                 continue;
     699             :                         }
     700             : 
     701      391982 :                         read_size = handle_read(&handle[j], block, buffer[j], state->block_size, log_error, 0);
     702      391982 :                         if (read_size == -1) {
     703             :                                 /* LCOV_EXCL_START */
     704             :                                 /* file we are processing for error reporting */
     705             :                                 struct snapraid_file* file = block_file_get(block);
     706             :                                 if (errno == EIO) {
     707             :                                         log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
     708             :                                         if (io_error >= state->opt.io_error_limit) {
     709             :                                                 log_fatal("DANGER! Unexpected input/output read error in a data disk, it isn't possible to sync.\n");
     710             :                                                 log_fatal("Ensure that disk '%s' is sane and that file '%s' can be read.\n", disk->dir, handle[j].path);
     711             :                                                 log_fatal("Stopping at block %u\n", i);
     712             :                                                 ++io_error;
     713             :                                                 goto bail;
     714             :                                         }
     715             : 
     716             :                                         log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
     717             :                                         ++io_error;
     718             :                                         io_error_on_this_block = 1;
     719             :                                         continue;
     720             :                                 }
     721             : 
     722             :                                 log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
     723             :                                 log_fatal("WARNING! Unexpected read error in a data disk, it isn't possible to sync.\n");
     724             :                                 log_fatal("Ensure that file '%s' can be read.\n", handle[j].path);
     725             :                                 log_fatal("Stopping to allow recovery. Try with 'snapraid check -f %s'\n", file->sub);
     726             :                                 ++error;
     727             :                                 goto bail;
     728             :                                 /* LCOV_EXCL_STOP */
     729             :                         }
     730             : 
     731             :                         /* until now is disk */
     732      391982 :                         state_usage_disk(state, disk);
     733             : 
     734      391982 :                         countsize += read_size;
     735             : 
     736             :                         /* now compute the hash */
     737      391982 :                         if (rehash) {
     738       52257 :                                 memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
     739             : 
     740             :                                 /* compute the new hash, and store it */
     741       52257 :                                 rehandle[j].block = block;
     742       52257 :                                 memhash(state->hash, state->hashseed, rehandle[j].hash, buffer[j], read_size);
     743             :                         } else {
     744      339725 :                                 memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
     745             :                         }
     746             : 
     747      391982 :                         if (block_has_updated_hash(block)) {
     748             :                                 /* compare the hash */
     749      272206 :                                 if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
     750             :                                         /* file we are processing for error reporting */
     751         884 :                                         struct snapraid_file* file = block_file_get(block);
     752             : 
     753             :                                         /* if the file has invalid parity, it's a REP changed during the sync */
     754         884 :                                         if (block_has_invalid_parity(block)) {
     755           1 :                                                 log_tag("error:%u:%s:%s: Unexpected data change\n", i, disk->name, esc(file->sub));
     756           1 :                                                 log_error("Data change at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
     757           1 :                                                 log_error("WARNING! Unexpected data modification of a file without parity!\n");
     758             : 
     759           1 :                                                 if (file_flag_has(file, FILE_IS_COPY)) {
     760           1 :                                                         log_error("This file was detected as a copy of another file with the same name, size,\n");
     761           1 :                                                         log_error("and timestamp, but the file data isn't matching the assumed copy.\n");
     762           1 :                                                         log_error("If this is a false positive, and the files are expected to be different,\n");
     763           1 :                                                         log_error("you can 'sync' anyway using 'snapraid --force-nocopy sync'\n");
     764             :                                                 } else {
     765           0 :                                                         log_error("Try removing the file from the array and rerun the 'sync' command!\n");
     766             :                                                 }
     767             : 
     768           1 :                                                 ++error;
     769             : 
     770             :                                                 /* if the file is changed, it means that it was modified during sync */
     771             :                                                 /* this isn't a serious error, so we skip this block, and continue with others */
     772           1 :                                                 error_on_this_block = 1;
     773           1 :                                                 continue;
     774             :                                         } else { /* otherwise it's a BLK with silent error */
     775         883 :                                                 log_tag("error:%u:%s:%s: Data error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block));
     776         883 :                                                 log_error("Data error in file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
     777             : 
     778             :                                                 /* save the failed block for the fix */
     779         883 :                                                 failed[failed_count].index = j;
     780         883 :                                                 failed[failed_count].size = read_size;
     781         883 :                                                 failed[failed_count].block = block;
     782         883 :                                                 ++failed_count;
     783             : 
     784             :                                                 /* silent errors are very rare, and are not a signal that a disk */
     785             :                                                 /* is going to fail. So, we just continue marking the block as bad */
     786             :                                                 /* just like in scrub */
     787         883 :                                                 ++silent_error;
     788         883 :                                                 silent_error_on_this_block = 1;
     789         883 :                                                 continue;
     790             :                                         }
     791             :                                 }
     792             :                         } else {
     793             :                                 /* if until now the parity doesn't need to be updated */
     794      119776 :                                 if (!parity_needs_to_be_updated) {
     795             :                                         /* for sure it's a CHG block, because EMPTY are processed before with "continue" */
     796             :                                         /* and BLK and REP have "block_has_updated_hash()" as 1, and all the others */
     797             :                                         /* have "parity_needs_to_be_updated" already at 1 */
     798       54370 :                                         assert(block_state_get(block) == BLOCK_STATE_CHG);
     799             : 
     800             :                                         /* if there is an hash */
     801       54370 :                                         if (hash_is_real(block->hash)) {
     802             :                                                 /* check if the hash is changed */
     803       10710 :                                                 if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
     804             :                                                         /* the block is different, and we must update parity */
     805        9515 :                                                         parity_needs_to_be_updated = 1;
     806             :                                                 }
     807             :                                         } else {
     808             :                                                 /* if the hash is already invalid, we update parity */
     809       43660 :                                                 parity_needs_to_be_updated = 1;
     810             :                                         }
     811             :                                 }
     812             : 
     813             :                                 /* copy the hash in the block, but doesn't mark the block as hashed */
     814             :                                 /* this allow in case of skipped block to do not save the failed computation */
     815      119776 :                                 memcpy(block->hash, hash, HASH_SIZE);
     816             : 
     817             :                                 /* note that in case of rehash, this is the wrong hash, */
     818             :                                 /* but it will be overwritten later */
     819             :                         }
     820             :                 }
     821             : 
     822             :                 /* if we have only silent errors we can try to fix them on-the-fly */
     823             :                 /* note the the fix is not written to disk, but used only to */
     824             :                 /* compute the new parity */
     825       79938 :                 if (!error_on_this_block && !io_error_on_this_block && silent_error_on_this_block) {
     826             :                         unsigned failed_mac;
     827         883 :                         int something_to_recover = 0;
     828             : 
     829             :                         /* setup the blocks to recover */
     830         883 :                         failed_mac = 0;
     831        2649 :                         for (j = 0; j < failed_count; ++j) {
     832        1766 :                                 unsigned char* block_buffer = buffer[failed[j].index];
     833        1766 :                                 unsigned char* block_copy = buffer[diskmax + state->level + failed[j].index];
     834        1766 :                                 unsigned block_state = block_state_get(failed[j].block);
     835             : 
     836             :                                 /* we try to recover only if at least one BLK is present */
     837        1766 :                                 if (block_state == BLOCK_STATE_BLK)
     838         883 :                                         something_to_recover = 1;
     839             : 
     840             :                                 /* save a copy of the content just read */
     841             :                                 /* that it's going to be overwritten by the recovering function */
     842        1766 :                                 memcpy(block_copy, block_buffer, state->block_size);
     843             : 
     844        1766 :                                 if (block_state == BLOCK_STATE_CHG
     845         883 :                                         && hash_is_zero(failed[j].block->hash)
     846             :                                 ) {
     847             :                                         /* if the block was filled with 0, restore this state */
     848             :                                         /* and avoid to recover it */
     849           0 :                                         memset(block_buffer, 0, state->block_size);
     850             :                                 } else {
     851             :                                         /* if we have too many failures, we cannot recover */
     852        1766 :                                         if (failed_mac >= state->level)
     853           0 :                                                 break;
     854             : 
     855             :                                         /* otherwise it has to be recovered */
     856        1766 :                                         failed_map[failed_mac++] = failed[j].index;
     857             :                                 }
     858             :                         }
     859             : 
     860             :                         /* if we have something to recover and enough parity */
     861         883 :                         if (something_to_recover && j == failed_count) {
     862             :                                 /* until now is CPU */
     863         883 :                                 state_usage_cpu(state);
     864             : 
     865             :                                 /* read the parity */
     866             :                                 /* we are sure that parity exists because */
     867             :                                 /* we have at least one BLK block */
     868        6181 :                                 for (l = 0; l < state->level; ++l) {
     869        5298 :                                         ret = parity_read(parity[l], i, buffer[diskmax + l], state->block_size, log_error);
     870        5298 :                                         if (ret == -1) {
     871             :                                                 /* LCOV_EXCL_START */
     872             :                                                 if (errno == EIO) {
     873             :                                                         log_tag("parity_error:%u:%s: Read EIO error. %s\n", i, lev_config_name(l), strerror(errno));
     874             :                                                         if (io_error >= state->opt.io_error_limit) {
     875             :                                                                 log_fatal("DANGER! Unexpected input/output read error in the %s disk, it isn't possible to sync.\n", lev_name(l));
     876             :                                                                 log_fatal("Ensure that disk '%s' is sane and can be read.\n", lev_config_name(l));
     877             :                                                                 log_fatal("Stopping at block %u\n", i);
     878             :                                                                 ++io_error;
     879             :                                                                 goto bail;
     880             :                                                         }
     881             : 
     882             :                                                         log_error("Input/Output error in parity '%s' at position '%u'\n", lev_config_name(l), i);
     883             :                                                         ++io_error;
     884             :                                                         io_error_on_this_block = 1;
     885             :                                                         continue;
     886             :                                                 }
     887             : 
     888             :                                                 log_tag("parity_error:%u:%s: Read error. %s\n", i, lev_config_name(l), strerror(errno));
     889             :                                                 log_fatal("WARNING! Unexpected read error in the %s disk, it isn't possible to sync.\n", lev_name(l));
     890             :                                                 log_fatal("Ensure that disk '%s' can be read.\n", lev_config_name(l));
     891             :                                                 log_fatal("Stopping at block %u\n", i);
     892             :                                                 ++error;
     893             :                                                 goto bail;
     894             :                                                 /* LCOV_EXCL_STOP */
     895             :                                         }
     896             : 
     897             :                                         /* until now is parity */
     898        5298 :                                         state_usage_parity(state, l);
     899             :                                 }
     900             : 
     901             :                                 /* if no error in parity read */
     902         883 :                                 if (!io_error_on_this_block) {
     903             :                                         /* try to fix the data */
     904             :                                         /* note that this is a simple fix algorithm, that doesn't take into */
     905             :                                         /* account the case of a wrong parity */
     906             :                                         /* only 'fix' supports the most advanced fixing */
     907         883 :                                         raid_rec(failed_mac, failed_map, diskmax, state->level, state->block_size, buffer);
     908             : 
     909             :                                         /* check the result and prepare the data */
     910        5298 :                                         for (j = 0; j < failed_count; ++j) {
     911             :                                                 unsigned char hash[HASH_SIZE];
     912        1766 :                                                 unsigned char* block_buffer = buffer[failed[j].index];
     913        1766 :                                                 unsigned char* block_copy = buffer[diskmax + state->level + failed[j].index];
     914        1766 :                                                 unsigned block_state = block_state_get(failed[j].block);
     915             : 
     916        1766 :                                                 if (block_state == BLOCK_STATE_BLK) {
     917         883 :                                                         unsigned size = failed[j].size;
     918             : 
     919             :                                                         /* compute the hash of the recovered block */
     920         883 :                                                         if (rehash) {
     921           0 :                                                                 memhash(state->prevhash, state->prevhashseed, hash, block_buffer, size);
     922             :                                                         } else {
     923         883 :                                                                 memhash(state->hash, state->hashseed, hash, block_buffer, size);
     924             :                                                         }
     925             : 
     926             :                                                         /* if the hash doesn't match */
     927         883 :                                                         if (memcmp(hash, failed[j].block->hash, HASH_SIZE) != 0) {
     928             :                                                                 /* we have not recovered */
     929           0 :                                                                 break;
     930             :                                                         }
     931             : 
     932             :                                                         /* pad with 0 if needed */
     933         883 :                                                         if (size < state->block_size)
     934         340 :                                                                 memset(block_buffer + size, 0, state->block_size - size);
     935             :                                                 } else {
     936             :                                                         /* otherwise restore the content */
     937             :                                                         /* because we are not interested in the old state */
     938             :                                                         /* that it's recovered for CHG, REP and DELETED blocks */
     939         883 :                                                         memcpy(block_buffer, block_copy, state->block_size);
     940             :                                                 }
     941             :                                         }
     942             : 
     943             :                                         /* if all is processed, we have fixed it */
     944         883 :                                         if (j == failed_count)
     945         883 :                                                 fixed_error_on_this_block = 1;
     946             :                                 }
     947             :                         }
     948             :                 }
     949             : 
     950             :                 /* if we have read all the data required and it's correct, proceed with the parity */
     951       79938 :                 if (!error_on_this_block && !io_error_on_this_block
     952       79919 :                         && (!silent_error_on_this_block || fixed_error_on_this_block)
     953             :                 ) {
     954             :                         /* updates the parity only if really needed */
     955       79919 :                         if (parity_needs_to_be_updated) {
     956             :                                 /* compute the parity */
     957       78964 :                                 raid_gen(diskmax, state->level, state->block_size, buffer);
     958             : 
     959             :                                 /* until now is CPU */
     960       78964 :                                 state_usage_cpu(state);
     961             : 
     962             :                                 /* write the parity */
     963      516438 :                                 for (l = 0; l < state->level; ++l) {
     964      437474 :                                         ret = parity_write(parity[l], i, buffer[diskmax + l], state->block_size);
     965      437474 :                                         if (ret == -1) {
     966             :                                                 /* LCOV_EXCL_START */
     967             :                                                 if (errno == EIO) {
     968             :                                                         log_tag("parity_error:%u:%s: Write EIO error. %s\n", i, lev_config_name(l), strerror(errno));
     969             :                                                         if (io_error >= state->opt.io_error_limit) {
     970             :                                                                 log_fatal("DANGER! Unexpected input/output write error in the %s disk, it isn't possible to sync.\n", lev_name(l));
     971             :                                                                 log_fatal("Ensure that disk '%s' is sane and can be written.\n", lev_config_name(l));
     972             :                                                                 log_fatal("Stopping at block %u\n", i);
     973             :                                                                 ++io_error;
     974             :                                                                 goto bail;
     975             :                                                         }
     976             : 
     977             :                                                         log_error("Input/Output error in parity '%s' at position '%u'\n", lev_config_name(l), i);
     978             :                                                         ++io_error;
     979             :                                                         io_error_on_this_block = 1;
     980             :                                                         continue;
     981             :                                                 }
     982             : 
     983             :                                                 log_tag("parity_error:%u:%s: Write error. %s\n", i, lev_config_name(l), strerror(errno));
     984             :                                                 log_fatal("WARNING! Unexpected write error in the %s disk, it isn't possible to sync.\n", lev_name(l));
     985             :                                                 log_fatal("Ensure that disk '%s' has some free space available.\n", lev_config_name(l));
     986             :                                                 log_fatal("Stopping at block %u\n", i);
     987             :                                                 ++error;
     988             :                                                 goto bail;
     989             :                                                 /* LCOV_EXCL_STOP */
     990             :                                         }
     991             : 
     992             :                                         /* until now is parity */
     993      437474 :                                         state_usage_parity(state, l);
     994             :                                 }
     995             :                         }
     996             : 
     997             :                         /* if no error in parity write */
     998       79919 :                         if (!io_error_on_this_block) {
     999             :                                 /* for each disk, mark the blocks as processed */
    1000      559433 :                                 for (j = 0; j < diskmax; ++j) {
    1001      479514 :                                         struct snapraid_block* block = BLOCK_EMPTY;
    1002      479514 :                                         if (handle[j].disk)
    1003      479514 :                                                 block = disk_block_get(handle[j].disk, i);
    1004             : 
    1005      479514 :                                         if (block == BLOCK_EMPTY) {
    1006             :                                                 /* nothing to do */
    1007       76559 :                                                 continue;
    1008             :                                         }
    1009             : 
    1010             :                                         /* if it's a deleted block */
    1011      402955 :                                         if (block_state_get(block) == BLOCK_STATE_DELETED) {
    1012             :                                                 /* the parity is now updated without this block, so it's now empty */
    1013       11067 :                                                 tommy_arrayblk_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY);
    1014       11067 :                                                 continue;
    1015             :                                         }
    1016             : 
    1017             :                                         /* now all the blocks have the hash and the parity computed */
    1018      391888 :                                         block_state_set(block, BLOCK_STATE_BLK);
    1019             :                                 }
    1020             :                         }
    1021             : 
    1022             :                         /* we update the info block only if we really have updated the parity */
    1023             :                         /* because otherwise the time info would be misleading as we didn't */
    1024             :                         /* wrote the parity at this time */
    1025             :                         /* we also update the info block only if no silent error was found */
    1026             :                         /* because has no sense to refresh the time for data that we know bad */
    1027       79919 :                         if (parity_needs_to_be_updated
    1028       78964 :                                 && !silent_error_on_this_block
    1029       78081 :                                 && !io_error_on_this_block
    1030             :                         ) {
    1031             :                                 /* if rehash is neeed */
    1032       78081 :                                 if (rehash) {
    1033             :                                         /* store all the new hash already computed */
    1034       64372 :                                         for (j = 0; j < diskmax; ++j) {
    1035       55176 :                                                 if (rehandle[j].block)
    1036       52257 :                                                         memcpy(rehandle[j].block->hash, rehandle[j].hash, HASH_SIZE);
    1037             :                                         }
    1038             :                                 }
    1039             : 
    1040             :                                 /* update the time info of the block */
    1041             :                                 /* we are also clearing any previous bad and rehash flag */
    1042       78081 :                                 info_set(&state->infoarr, i, info_make(now, 0, 0));
    1043             :                         }
    1044             :                 }
    1045             : 
    1046             :                 /* if a silent (even if corrected) or input/output error was found */
    1047             :                 /* mark the block as bad to have check/fix to handle it */
    1048             :                 /* because our correction is in memory only and not yet written */
    1049       79938 :                 if (silent_error_on_this_block || io_error_on_this_block) {
    1050             :                         /* set the error status keeping the other info */
    1051         883 :                         info_set(&state->infoarr, i, info_set_bad(info));
    1052             :                 }
    1053             : 
    1054             :                 /* mark the state as needing write */
    1055       79938 :                 state->need_write = 1;
    1056             : 
    1057             :                 /* count the number of processed block */
    1058       79938 :                 ++countpos;
    1059             : 
    1060             :                 /* progress */
    1061       79938 :                 if (state_progress(state, i, countpos, countmax, countsize)) {
    1062             :                         /* LCOV_EXCL_START */
    1063             :                         break;
    1064             :                         /* LCOV_EXCL_STOP */
    1065             :                 }
    1066             : 
    1067             :                 /* autosave */
    1068       79938 :                 if ((state->autosave != 0
    1069        7262 :                         && autosavedone >= autosavelimit /* if we have reached the limit */
    1070           0 :                         && autosavemissing >= autosavelimit) /* if we have at least a full step to do */
    1071             :                         /* or if we have a forced autosave at the specified block */
    1072       79938 :                         || (state->opt.force_autosave_at != 0 && state->opt.force_autosave_at == i)
    1073             :                 ) {
    1074           1 :                         autosavedone = 0; /* restart the counter */
    1075             : 
    1076             :                         /* until now is CPU */
    1077           1 :                         state_usage_cpu(state);
    1078             : 
    1079           1 :                         state_progress_stop(state);
    1080             : 
    1081           1 :                         msg_progress("Autosaving...\n");
    1082             : 
    1083             :                         /* before writing the new content file we ensure that */
    1084             :                         /* the parity is really written flushing the disk cache */
    1085           7 :                         for (l = 0; l < state->level; ++l) {
    1086           6 :                                 ret = parity_sync(parity[l]);
    1087           6 :                                 if (ret == -1) {
    1088             :                                         /* LCOV_EXCL_START */
    1089             :                                         log_tag("parity_error:%u:%s: Sync error\n", i, lev_config_name(l));
    1090             :                                         log_fatal("DANGER! Unexpected sync error in %s disk.\n", lev_name(l));
    1091             :                                         log_fatal("Ensure that disk '%s' is sane.\n", lev_config_name(l));
    1092             :                                         log_fatal("Stopping at block %u\n", i);
    1093             :                                         ++error;
    1094             :                                         goto bail;
    1095             :                                         /* LCOV_EXCL_STOP */
    1096             :                                 }
    1097             :                         }
    1098             : 
    1099             :                         /* now we can safely write the content file */
    1100           1 :                         state_write(state);
    1101             : 
    1102           1 :                         state_progress_restart(state);
    1103             : 
    1104             :                         /* drop until now */
    1105           1 :                         state_usage_waste(state);
    1106             :                 }
    1107             :         }
    1108             : 
    1109             : end:
    1110          62 :         state_progress_end(state, countpos, countmax, countsize);
    1111             : 
    1112          62 :         state_usage_print(state);
    1113             : 
    1114             :         /* before returning we ensure that */
    1115             :         /* the parity is really written flushing the disk cache */
    1116         424 :         for (l = 0; l < state->level; ++l) {
    1117         362 :                 ret = parity_sync(parity[l]);
    1118         362 :                 if (ret == -1) {
    1119             :                         /* LCOV_EXCL_START */
    1120             :                         log_tag("parity_error:%u:%s: Sync error\n", i, lev_config_name(l));
    1121             :                         log_fatal("DANGER! Unexpected sync error in %s disk.\n", lev_name(l));
    1122             :                         log_fatal("Ensure that disk '%s' is sane.\n", lev_config_name(l));
    1123             :                         log_fatal("Stopping at block %u\n", i);
    1124             :                         ++error;
    1125             :                         goto bail;
    1126             :                         /* LCOV_EXCL_STOP */
    1127             :                 }
    1128             :         }
    1129             : 
    1130          62 :         if (error || silent_error || io_error) {
    1131          12 :                 msg_status("\n");
    1132          12 :                 msg_status("%8u file errors\n", error);
    1133          12 :                 msg_status("%8u io errors\n", io_error);
    1134          12 :                 msg_status("%8u data errors\n", silent_error);
    1135             :         } else {
    1136             :                 /* print the result only if processed something */
    1137          50 :                 if (countpos != 0)
    1138          36 :                         msg_status("Everything OK\n");
    1139             :         }
    1140             : 
    1141          62 :         if (error)
    1142          11 :                 log_fatal("WARNING! Unexpected file errors!\n");
    1143          62 :         if (io_error)
    1144           0 :                 log_fatal("DANGER! Unexpected input/output errors! The failing blocks are now marked as bad!\n");
    1145          62 :         if (silent_error)
    1146           1 :                 log_fatal("DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n");
    1147          62 :         if (io_error || silent_error) {
    1148           1 :                 log_fatal("Use 'snapraid status' to list the bad blocks.\n");
    1149           1 :                 log_fatal("Use 'snapraid -e fix' to recover.\n");
    1150             :         }
    1151             : 
    1152          62 :         log_tag("summary:error_file:%u\n", error);
    1153          62 :         log_tag("summary:error_io:%u\n", io_error);
    1154          62 :         log_tag("summary:error_data:%u\n", silent_error);
    1155          62 :         if (error + silent_error + io_error == 0)
    1156          50 :                 log_tag("summary:exit:ok\n");
    1157             :         else
    1158          12 :                 log_tag("summary:exit:error\n");
    1159          62 :         log_flush();
    1160             : 
    1161             : bail:
    1162         434 :         for (j = 0; j < diskmax; ++j) {
    1163         372 :                 struct snapraid_file* file = handle[j].file;
    1164         372 :                 struct snapraid_disk* disk = handle[j].disk;
    1165         372 :                 ret = handle_close(&handle[j]);
    1166         372 :                 if (ret == -1) {
    1167             :                         /* LCOV_EXCL_START */
    1168             :                         log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
    1169             :                         log_fatal("DANGER! Unexpected close error in a data disk.\n");
    1170             :                         ++error;
    1171             :                         /* continue, as we are already exiting */
    1172             :                         /* LCOV_EXCL_STOP */
    1173             :                 }
    1174             :         }
    1175             : 
    1176          62 :         free(handle);
    1177          62 :         free(buffer_alloc);
    1178          62 :         free(buffer);
    1179          62 :         free(rehandle_alloc);
    1180          62 :         free(failed);
    1181          62 :         free(failed_map);
    1182             : 
    1183          62 :         if (state->opt.expect_recoverable) {
    1184           1 :                 if (error + silent_error + io_error == 0)
    1185           0 :                         return -1;
    1186             :         } else {
    1187          61 :                 if (error + silent_error + io_error != 0)
    1188          11 :                         return -1;
    1189             :         }
    1190          51 :         return 0;
    1191             : }
    1192             : 
    1193          63 : int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount)
    1194             : {
    1195             :         block_off_t blockmax;
    1196             :         block_off_t used_paritymax;
    1197             :         block_off_t file_paritymax;
    1198             :         data_off_t size;
    1199             :         data_off_t out_size;
    1200             :         int ret;
    1201             :         struct snapraid_parity_handle parity[LEV_MAX];
    1202             :         /* the following initialization is to avoid clang warnings about */
    1203             :         /* potential state->level change, that never happens */
    1204          63 :         struct snapraid_parity_handle* parity_ptr[LEV_MAX] = { 0 };
    1205             :         unsigned unrecoverable_error;
    1206             :         unsigned l;
    1207          63 :         int skip_sync = 0;
    1208             : 
    1209          63 :         msg_progress("Initializing...\n");
    1210             : 
    1211          63 :         blockmax = parity_allocated_size(state);
    1212          63 :         size = blockmax * (data_off_t)state->block_size;
    1213             : 
    1214             :         /* minum size of the parity files we expect */
    1215          63 :         used_paritymax = parity_used_size(state);
    1216             : 
    1217             :         /* effective size of the parity files */
    1218          63 :         file_paritymax = 0;
    1219             : 
    1220          63 :         if (blockstart > blockmax) {
    1221             :                 /* LCOV_EXCL_START */
    1222             :                 log_fatal("Error in the starting block %u. It's bigger than the parity size %u.\n", blockstart, blockmax);
    1223             :                 exit(EXIT_FAILURE);
    1224             :                 /* LCOV_EXCL_STOP */
    1225             :         }
    1226             : 
    1227             :         /* adjust the number of block to process */
    1228          63 :         if (blockcount != 0 && blockstart + blockcount < blockmax) {
    1229           5 :                 blockmax = blockstart + blockcount;
    1230             :         }
    1231             : 
    1232         431 :         for (l = 0; l < state->level; ++l) {
    1233             :                 block_off_t parityblocks;
    1234             : 
    1235             :                 /* create the file and open for writing */
    1236         368 :                 parity_ptr[l] = &parity[l];
    1237         368 :                 ret = parity_create(parity_ptr[l], state->parity[l].path, &out_size, state->file_mode);
    1238         368 :                 if (ret == -1) {
    1239             :                         /* LCOV_EXCL_START */
    1240             :                         log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
    1241             :                         exit(EXIT_FAILURE);
    1242             :                         /* LCOV_EXCL_STOP */
    1243             :                 }
    1244             : 
    1245             :                 /* number of block in the parity file */
    1246         368 :                 parityblocks = out_size / state->block_size;
    1247             : 
    1248             :                 /* if the file is too small */
    1249         368 :                 if (parityblocks < used_paritymax) {
    1250           0 :                         log_fatal("WARNING! The %s file %s has data only %u blocks instead of %u.\n", lev_name(l), state->parity[l].path, parityblocks, used_paritymax);
    1251             :                 }
    1252             : 
    1253             :                 /* keep the smallest parity number of blocks */
    1254         368 :                 if (l == 0 || file_paritymax > parityblocks)
    1255          63 :                         file_paritymax = parityblocks;
    1256             :         }
    1257             : 
    1258             :         /* if we do a full sync, having a wrong parity size is expected */
    1259          63 :         if (!state->opt.force_full) {
    1260             :                 /* if the parities are too small */
    1261          62 :                 if (file_paritymax < used_paritymax) {
    1262             :                         /* LCOV_EXCL_START */
    1263             :                         log_fatal("DANGER! One or more the parity files are smaller than expected!\n");
    1264             :                         if (file_paritymax != 0) {
    1265             :                                 log_fatal("If this happens because you are using an old content file,\n");
    1266             :                                 log_fatal("you can 'sync' anyway using 'snapraid --force-full sync'\n");
    1267             :                                 log_fatal("to force a full rebuild of the parity.\n");
    1268             :                         } else {
    1269             :                                 log_fatal("It's possible that the parity disks are not mounted.\n");
    1270             :                                 log_fatal("If instead you are adding a new parity level, you can 'sync' using\n");
    1271             :                                 log_fatal("'snapraid --force-full sync' to force a full rebuild of the parity.\n");
    1272             :                         }
    1273             :                         exit(EXIT_FAILURE);
    1274             :                         /* LCOV_EXCL_STOP */
    1275             :                 }
    1276             :         }
    1277             : 
    1278             :         /* now change the size of all parities */
    1279         431 :         for (l = 0; l < state->level; ++l) {
    1280             :                 /* change the size of the parity file, truncating or extending it */
    1281             :                 /* from this point all the DELETED blocks after the end of the parity are invalid */
    1282             :                 /* and they are automatically removed when we save the new content file */
    1283         368 :                 ret = parity_chsize(parity_ptr[l], size, &out_size, state->opt.skip_fallocate);
    1284         368 :                 if (ret == -1) {
    1285             :                         /* LCOV_EXCL_START */
    1286             :                         parity_overflow(state, out_size);
    1287             :                         log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
    1288             :                         exit(EXIT_FAILURE);
    1289             :                         /* LCOV_EXCL_STOP */
    1290             :                 }
    1291             :         }
    1292             : 
    1293             :         /* after resizing parity files, refresh again the free info */
    1294          63 :         state_refresh(state);
    1295             : 
    1296          63 :         unrecoverable_error = 0;
    1297             : 
    1298          63 :         if (state->opt.prehash) {
    1299           6 :                 msg_progress("Hashing...\n");
    1300             : 
    1301           6 :                 ret = state_hash_process(state, blockstart, blockmax, &skip_sync);
    1302           6 :                 if (ret == -1) {
    1303             :                         /* LCOV_EXCL_START */
    1304             :                         ++unrecoverable_error;
    1305             :                         /* continue, in case also doing the sync */
    1306             :                         /* LCOV_EXCL_STOP */
    1307             :                 }
    1308             : 
    1309           6 :                 if (state->need_write)
    1310           3 :                         state_write(state);
    1311             :         }
    1312             : 
    1313          63 :         if (!skip_sync) {
    1314          63 :                 msg_progress("Syncing...\n");
    1315             : 
    1316             :                 /* skip degenerated cases of empty parity, or skipping all */
    1317          63 :                 if (blockstart < blockmax) {
    1318          62 :                         ret = state_sync_process(state, parity_ptr, blockstart, blockmax);
    1319          62 :                         if (ret == -1) {
    1320             :                                 /* LCOV_EXCL_START */
    1321             :                                 ++unrecoverable_error;
    1322             :                                 /* continue, as we are already exiting */
    1323             :                                 /* LCOV_EXCL_STOP */
    1324             :                         }
    1325             :                 } else {
    1326           1 :                         msg_status("Nothing to do\n");
    1327             :                 }
    1328             :         }
    1329             : 
    1330         431 :         for (l = 0; l < state->level; ++l) {
    1331         368 :                 ret = parity_close(parity_ptr[l]);
    1332         368 :                 if (ret == -1) {
    1333             :                         /* LCOV_EXCL_START */
    1334             :                         log_fatal("DANGER! Unexpected close error in %s disk.\n", lev_name(l));
    1335             :                         ++unrecoverable_error;
    1336             :                         /* continue, as we are already exiting */
    1337             :                         /* LCOV_EXCL_STOP */
    1338             :                 }
    1339             :         }
    1340             : 
    1341             :         /* abort if required */
    1342          63 :         if (unrecoverable_error != 0)
    1343          11 :                 return -1;
    1344          52 :         return 0;
    1345             : }
    1346             : 

Generated by: LCOV version 1.10