/* * Copyright (C) 2015 Andrea Mazzoleni * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "portable.h" #include "support.h" #include "state.h" #include "raid/raid.h" /** * Annual Failure Rate data points from Backblaze. * * From: https://www.backblaze.com/blog-smart-stats-2014-8.html * Mentioned link for future update: https://www.backblaze.com/smart */ struct afr_point { uint64_t value; /**< Value of the SMART raw attribute. */ double afr; /**< Annual Failure Rate at this value. */ }; /* Smart 5: Reallocated_Sector_Ct (Raw Value) */ static struct afr_point AFR_5[] = { { 0, 0 }, { (1+4)/2, 0.0274 }, { (4+16)/2, 0.0750 }, { (16+70)/2, 0.235 }, { (70+260)/2, 0.361 }, { (260+1100)/2, 0.567 }, { (1100+4500)/2, 1.50 }, { (4500+17000)/2, 2.06 }, { (17000+70000)/2, 1.77 }, { 0, 0 } }; /* Smart 187: Reported_Uncorrect (Raw Value) */ static struct afr_point AFR_187[] = { { 0, 0 }, { 1, 0.338 }, { (3+6)/2, 0.501 }, { (11+20)/2, 0.534 }, { (20+35)/2, 0.842 }, { (35+65)/2, 1.44 }, { (65+120)/2, 1.61 }, { 0, 0 } }; /* Smart 188: Command_Timeout (Raw Value) */ static struct afr_point AFR_188[] = { { 0, 0 }, { (1+13000000000ULL)/2, 0.100 }, { (13000000000ULL+26000000000ULL)/2, 0.334 }, { (26000000000ULL+104000000000ULL)/2, 0.367 }, { 0, 0 } }; /* Smart 193: Load_Cycle_Count (Raw Value) */ static struct afr_point AFR_193[] = { { 0, 0 }, { (1300+5500)/2, 0.0248 }, { (5500+21000)/2, 0.0585 }, { (21000+90000)/2, 0.195 }, { (90000+350000)/2, 0.267 }, { 0, 0 } }; /* Smart 197: Current_Pending_Sector (Raw Value) */ static struct afr_point AFR_197[] = { { 0, 0 }, { (1+2)/2, 0.341 }, { (2+6)/2, 0.682 }, { (16+40)/2, 0.956 }, { (40+100)/2, 1.65 }, { (100+250)/2, 2.51 }, { (250+650)/2, 3.32 }, { 0, 0 } }; /* Smart 198: Offline_Uncorrectable (Raw Value) */ static struct afr_point AFR_198[] = { { 0, 0 }, { (1+2)/2, 0.813 }, { (2+4)/2, 1.11 }, { (4+6)/2, 1.35 }, { (10+12)/2, 1.74 }, { (12+14)/2, 2.64 }, { 0, 0 } }; /** * Computes the estimated AFR (Annual Failure Rate) from a set of data points. */ static double smart_afr_value(struct afr_point* tab, uint64_t value) { unsigned i; double delta_afr; double delta_value; /* if first point */ if (value == 0) return 0; i = 1; while (tab[i].value != 0 && tab[i].value < value) ++i; /* if last point */ if (tab[i].value == 0) return tab[i-1].afr; /* if exact value */ if (tab[i].value == value) return tab[i].afr; delta_afr = tab[i].afr - tab[i-1].afr; delta_value = tab[i].value - tab[i-1].value; /* linear interpolation between the two points */ return tab[i-1].afr + (value - tab[i-1].value) * delta_afr / delta_value; } /** * Computes the estimated AFR of a set of SMART attributes. * * We assume the AFR (Annual Failure Rate) data from Backblaze * defined as AFR = 8760/MTBF (Mean Time Between Failures in hours). * * This is consistent with their definition of AFR: * "An annual failure rate of 100% means that if you have one disk drive slot * and keep a drive running in it all the time, you can expect an average of * one failure a year. If, on the other hand, you have one failure per month * in your one drive slot, then your failure rate is 1200%. If you run n * drives for t years with an annual failure rate of r, the number of failures * is expected to be n * r * t." * * Note that this definition is different from the one given * by Seagate, that defines AFR = 1 - exp(-8760/MTBF), that * instead represents the probability of a failure in the next * year. * * To combine the different AFR from different SMART attributes, * we sums them as we assume that they are independent, * (even if likely they are not). */ static double smart_afr(uint64_t* smart) { double afr = 0; if (smart[5] != SMART_UNASSIGNED) afr += smart_afr_value(AFR_5, smart[5]); if (smart[187] != SMART_UNASSIGNED) afr += smart_afr_value(AFR_187, smart[187]); if (smart[188] != SMART_UNASSIGNED) afr += smart_afr_value(AFR_188, smart[188]); if (smart[193] != SMART_UNASSIGNED) afr += smart_afr_value(AFR_193, smart[193]); if (smart[197] != SMART_UNASSIGNED) afr += smart_afr_value(AFR_197, smart[197]); if (smart[198] != SMART_UNASSIGNED) afr += smart_afr_value(AFR_198, smart[198]); return afr; } /** * Factorial. */ static double fact(unsigned n) { double v = 1; while (n > 1) v *= n--; return v; } /** * Probability of having exactly ::n events in a Poisson * distribution with rate ::rate in a time unit. */ static double poisson_prob_n_failures(double rate, unsigned n) { return pow(rate, n) * exp(-rate) / fact(n); } /** * Probability of having ::n or more events in a Poisson * distribution with rate ::rate in a time unit. */ static double poisson_prob_n_or_more_failures(double rate, unsigned n) { double p_neg = 0; unsigned i; for (i = 0; i < n; ++i) p_neg += poisson_prob_n_failures(rate, n - 1 - i); return 1 - p_neg; } /** * Probability of having data loss in a RAID system with the specified ::redundancy * supposing the specified ::array_failure_rate, and ::replace_rate. */ static double raid_prob_of_one_or_more_failures(double array_failure_rate, double replace_rate, unsigned n, unsigned redundancy) { unsigned i; double MTBF; double MTTR; double MTTDL; double raid_failure_rate; /* * Use the MTTDL model (Mean Time To Data Loss) to estimate the * failure rate of the array. * * See: * Garth Alan Gibson, "Redundant Disk Arrays: Reliable, Parallel Secondary Storage", 1990 */ /* get the Mean Time Between Failure of a single disk */ /* from the array failure rate */ MTBF = n / array_failure_rate; /* get the Mean Time Between Repair (the time that a failed disk is replaced) */ /* from the repair rate */ MTTR = 1.0 / replace_rate; /* use the approximated MTTDL equation */ MTTDL = pow(MTBF, redundancy + 1) / pow(MTTR, redundancy); for (i = 0; i < redundancy + 1; ++i) MTTDL /= n - i; /* the raid failure rate is just the inverse of the MTTDL */ raid_failure_rate = 1.0 / MTTDL; /* probability of at least one RAID failure */ /* note that is almost equal at the probabilty of */ /* the first failure. */ return poisson_prob_n_or_more_failures(raid_failure_rate, 1); } /** * Prints a string with space padding. */ static void printl(const char* str, size_t pad) { size_t len; printf("%s", str); len = strlen(str); while (len < pad) { printf(" "); ++len; } } /** * Prints a probability with space padding. */ static void printp(double v, size_t pad) { char buf[64]; const char* s = "%"; if (v > 0.1) snprintf(buf, sizeof(buf), "%5.2f%s", v, s); else if (v > 0.01) snprintf(buf, sizeof(buf), "%6.3f%s", v, s); else if (v > 0.001) snprintf(buf, sizeof(buf), "%7.4f%s", v, s); else if (v > 0.0001) snprintf(buf, sizeof(buf), "%8.5f%s", v, s); else if (v > 0.00001) snprintf(buf, sizeof(buf), "%9.6f%s", v, s); else if (v > 0.000001) snprintf(buf, sizeof(buf), "%10.7f%s", v, s); else if (v > 0.0000001) snprintf(buf, sizeof(buf), "%11.8f%s", v, s); else if (v > 0.00000001) snprintf(buf, sizeof(buf), "%12.9f%s", v, s); else if (v > 0.000000001) snprintf(buf, sizeof(buf), "%13.10f%s", v, s); else if (v > 0.0000000001) snprintf(buf, sizeof(buf), "%14.11f%s", v, s); else if (v > 0.00000000001) snprintf(buf, sizeof(buf), "%15.12f%s", v, s); else if (v > 0.000000000001) snprintf(buf, sizeof(buf), "%16.13f%s", v, s); else snprintf(buf, sizeof(buf), "%17.14f%s", v, s); printl(buf, pad); } static void state_smart(int verbose, unsigned n, tommy_list* low) { tommy_node* i; unsigned j; size_t device_pad; size_t serial_pad; int have_parent; double array_failure_rate; double p_at_least_one_failure; int make_it_fail = 0; /* compute lengths for padding */ device_pad = 0; serial_pad = 0; have_parent = 0; for (i = tommy_list_head(low); i != 0; i = i->next) { size_t len; devinfo_t* devinfo = i->data; len = strlen(devinfo->file); if (len > device_pad) device_pad = len; len = strlen(devinfo->smart_serial); if (len > serial_pad) serial_pad = len; if (devinfo->parent != 0) have_parent = 1; } printf("SnapRAID SMART report:\n"); printf("\n"); printf(" Temp"); printf(" Power"); printf(" Error"); printf(" FP"); printf(" Size"); printf("\n"); printf(" C"); printf(" OnDays"); printf(" Count"); printf(" "); printf(" TB"); printf(" "); printl("Serial", serial_pad); printf(" "); printl("Device", device_pad); printf(" Disk"); printf("\n"); /* |<##################################################################72>|####80>| */ printf(" -----------------------------------------------------------------------\n"); array_failure_rate = 0; for (i = tommy_list_head(low); i != 0; i = i->next) { devinfo_t* devinfo = i->data; double afr; uint64_t flag; if (devinfo->smart[194] != SMART_UNASSIGNED) printf("%7" PRIu64, devinfo->smart[194]); else if (devinfo->smart[190] != SMART_UNASSIGNED) printf("%7" PRIu64, devinfo->smart[190]); else printf(" -"); if (devinfo->smart[9] != SMART_UNASSIGNED) printf("%7" PRIu64, devinfo->smart[9] / 24); else printf(" -"); if (devinfo->smart[SMART_FLAGS] != SMART_UNASSIGNED) flag = devinfo->smart[SMART_FLAGS]; else flag = 0; if (flag & SMARTCTL_FLAG_FAIL) printf(" FAIL"); else if (flag & (SMARTCTL_FLAG_PREFAIL | SMARTCTL_FLAG_PREFAIL_LOGGED)) printf(" PREFAIL"); else if (devinfo->smart[SMART_ERROR] != SMART_UNASSIGNED && devinfo->smart[SMART_ERROR] != 0) printf("%8" PRIu64, devinfo->smart[SMART_ERROR]); else if (flag & (SMARTCTL_FLAG_ERROR | SMARTCTL_FLAG_ERROR_LOGGED)) printf(" ERROR"); else if (flag & (SMARTCTL_FLAG_UNSUPPORTED | SMARTCTL_FLAG_OPEN | SMARTCTL_FLAG_COMMAND)) printf(" -"); else if (devinfo->smart[SMART_ERROR] == 0) printf(" 0"); else printf(" -"); /* if some fail/prefail attribute, make the command to fail */ if (flag & (SMARTCTL_FLAG_FAIL | SMARTCTL_FLAG_PREFAIL | SMARTCTL_FLAG_PREFAIL_LOGGED)) make_it_fail = 1; if (flag & (SMARTCTL_FLAG_UNSUPPORTED | SMARTCTL_FLAG_OPEN | SMARTCTL_FLAG_COMMAND)) { /* if error running smartctl, skip AFR estimation */ afr = 0; printf(" -"); } else if (devinfo->smart[SMART_ROTATION_RATE] == 0) { /* if SSD, skip AFR estimation as data is from not SSD disks */ afr = 0; printf(" -"); } else { afr = smart_afr(devinfo->smart); /* use only afr of disks in the array */ if (devinfo->parent != 0 || !have_parent) array_failure_rate += afr; printf("%4.0f%%", poisson_prob_n_or_more_failures(afr, 1) * 100); } if (devinfo->smart[SMART_SIZE] != SMART_UNASSIGNED) printf(" %2.1f", devinfo->smart[SMART_SIZE] / 1E12); else printf(" -"); printf(" "); if (*devinfo->smart_serial) printl(devinfo->smart_serial, serial_pad); else printl("-", serial_pad); printf(" "); if (*devinfo->file) printl(devinfo->file, device_pad); else printl("-", device_pad); printf(" "); if (*devinfo->name) printf("%s", devinfo->name); else printf("-"); printf("\n"); ftag("smart:%s:%s:%s:%g\n", devinfo->file, devinfo->name, devinfo->smart_serial, afr); for (j = 0; j < 256; ++j) { if (devinfo->smart[j] != SMART_UNASSIGNED) ftag("attr:%s:%u:%" PRIu64 "\n", devinfo->file, j, devinfo->smart[j]); } if (devinfo->smart[SMART_SIZE] != SMART_UNASSIGNED) ftag("attr:%s:size:%" PRIu64 "\n", devinfo->file, devinfo->smart[SMART_SIZE]); if (devinfo->smart[SMART_ERROR] != SMART_UNASSIGNED) ftag("attr:%s:error:%" PRIu64 "\n", devinfo->file, devinfo->smart[SMART_ERROR]); if (devinfo->smart[SMART_ROTATION_RATE] != SMART_UNASSIGNED) ftag("attr:%s:rotationrate:%" PRIu64 "\n", devinfo->file, devinfo->smart[SMART_ROTATION_RATE]); } printf("\n"); /* * The probability of one and of at least one failure is computed assuming * a Poisson distribution with the estimated array failure rate. */ p_at_least_one_failure = poisson_prob_n_or_more_failures(array_failure_rate, 1); ftag("summary:array_failure:%g:%g\n", array_failure_rate, p_at_least_one_failure); /* |<##################################################################72>|####80>| */ printf("The FP column is the estimated probability (in percentage) that the disk\n"); printf("is going to fail in the next year.\n"); printf("\n"); printf("Probability that one disk is going to fail in the next year is: %.0f%%\n", p_at_least_one_failure * 100); /* prints extra stats only in verbose mode */ if (!verbose) return; /* |<##################################################################72>|####80>| */ printf("\n"); printf("Probability of data loss in the next year for different parity and\n"); printf("combined scrub and repair time:\n"); printf("\n"); printf(" Parity 1 Week 1 Month 3 Months\n"); printf(" -----------------------------------------------------------------------\n"); for (j = 0; j < RAID_PARITY_MAX; ++j) { printf("%6u", j + 1); printf(" "); printp(raid_prob_of_one_or_more_failures(array_failure_rate, 365.0 / 7, n, j + 1) * 100, 19); printf(" "); printp(raid_prob_of_one_or_more_failures(array_failure_rate, 365.0 / 30, n, j + 1) * 100, 17); printf(" "); printp(raid_prob_of_one_or_more_failures(array_failure_rate, 365.0 / 90, n, j + 1) * 100, 13); printf("\n"); } printf("\n"); /* |<##################################################################72>|####80>| */ printf("These values are the probabilities that in the next year you'll have a\n"); printf("sequence of failures that the parity WONT be able to recover, assuming\n"); printf("that you regularly scrub, and in case repair, the array in the specified\n"); printf("time.\n"); if (make_it_fail) { /* LCOV_EXCL_START */ printf("\n"); printf("DANGER! SMART is reporting that one or more disks are FAILING!\n"); printf("Please take immediate action!\n"); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } } void state_device(struct snapraid_state* state, int operation) { tommy_node* i; unsigned j; tommy_list high; tommy_list low; switch (operation) { case DEVICE_UP : printf("Spinup...\n"); break; case DEVICE_DOWN : printf("Spindown...\n"); break; } tommy_list_init(&high); tommy_list_init(&low); /* for all disks */ for (i = state->disklist; i != 0; i = i->next) { struct snapraid_disk* disk = i->data; devinfo_t* entry; entry = calloc_nofail(1, sizeof(devinfo_t)); entry->device = disk->device; pathcpy(entry->name, sizeof(entry->name), disk->name); pathcpy(entry->mount, sizeof(entry->mount), disk->dir); tommy_list_insert_tail(&high, &entry->node, entry); } /* for all parities */ for (j = 0; j < state->level; ++j) { devinfo_t* entry; entry = calloc_nofail(1, sizeof(devinfo_t)); entry->device = state->parity[j].device; pathcpy(entry->name, sizeof(entry->name), lev_config_name(j)); pathcpy(entry->mount, sizeof(entry->mount), state->parity[j].path); pathcut(entry->mount); /* remove the parity file */ tommy_list_insert_tail(&high, &entry->node, entry); } if (devquery(&high, &low, operation) != 0) { const char* ope = 0; switch (operation) { case DEVICE_UP : ope = "Spinup"; break; case DEVICE_DOWN : ope = "Spindown"; break; case DEVICE_LIST : ope = "List"; break; case DEVICE_SMART : ope = "SMART"; break; } ferr("%s unsupported in this platform.\n", ope); } else { #ifndef _WIN32 if (operation == DEVICE_LIST) { for (i = tommy_list_head(&low); i != 0; i = i->next) { devinfo_t* devinfo = i->data; devinfo_t* parent = devinfo->parent; printf("%u:%u\t%s\t%u:%u\t%s\t%s\n", major(devinfo->device), minor(devinfo->device), devinfo->file, major(parent->device), minor(parent->device), parent->file, parent->name); } } #endif if (operation == DEVICE_SMART) state_smart(state->opt.verbose, state->level + tommy_list_count(&state->disklist), &low); } tommy_list_foreach(&high, free); tommy_list_foreach(&low, free); }