// SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni #include "os/portable.h" #include "support.h" #include "util.h" #include "elem.h" #include "state.h" #include "parity.h" #include "handle.h" /****************************************************************************/ /* dup */ struct snapraid_hash { struct snapraid_disk* disk; /**< Disk. */ struct snapraid_file* file; /**< File. */ unsigned char hash[HASH_MAX]; /**< Hash of the whole file. */ /* nodes for data structures */ tommy_hashdyn_node node; }; static void hash_size_store(unsigned char* ptr, uint64_t size) { ptr[0] = (unsigned char)(size >> 0); ptr[1] = (unsigned char)(size >> 8); ptr[2] = (unsigned char)(size >> 16); ptr[3] = (unsigned char)(size >> 24); ptr[4] = (unsigned char)(size >> 32); ptr[5] = (unsigned char)(size >> 40); ptr[6] = (unsigned char)(size >> 48); ptr[7] = (unsigned char)(size >> 56); } struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) { struct snapraid_hash* hash; block_off_t i; unsigned char* buf; size_t hash_size = BLOCK_HASH_SIZE; size_t data_size = file->blockmax * hash_size; hash = malloc_nofail(sizeof(struct snapraid_hash)); hash->disk = disk; hash->file = file; buf = malloc_nofail(data_size + sizeof(uint64_t)); /* set the back pointer */ for (i = 0; i < file->blockmax; ++i) { struct snapraid_block* block = fs_file2block_get(file, i); hash_copy(buf + i * hash_size, block->hash); if (!block_has_updated_hash(block)) { free(buf); free(hash); return 0; } } hash_size_store(buf + data_size, (uint64_t)file->size); memhash(state->besthash, state->hashseed, hash->hash, buf, data_size + sizeof(uint64_t)); free(buf); return hash; } static inline tommy_hash_t hash_hash(struct snapraid_hash* hash) { #if TOMMY_SIZE_BIT == 64 return tommy_hash_u64(0, hash->hash, HASH_MAX); #else return tommy_hash_u32(0, hash->hash, HASH_MAX); #endif } static void hash_free(void* void_hash) { struct snapraid_hash* hash = void_hash; free(hash); } int hash_compare(const void* void_arg, const void* void_data) { const char* arg = void_arg; const struct snapraid_hash* hash = void_data; return memcmp(arg, hash->hash, HASH_MAX); } void state_dup(struct snapraid_state* state) { tommy_hashdyn hashset; tommy_node* i; unsigned count; data_off_t size; tommy_hashdyn_init(&hashset); count = 0; size = 0; msg_progress("Comparing...\n"); /* for each disk */ for (i = state->disklist; i != 0; i = i->next) { tommy_node* j; struct snapraid_disk* disk = i->data; /* for each file */ for (j = disk->filelist; j != 0; j = j->next) { struct snapraid_file* file = j->data; struct snapraid_hash* hash; tommy_hash_t hash_val; /* if empty, skip it */ if (file->size == 0) continue; hash = hash_alloc(state, disk, file); /* if no hash, skip it */ if (!hash) continue; hash_val = hash_hash(hash); struct snapraid_hash* found = tommy_hashdyn_search(&hashset, hash_compare, hash->hash, hash_val); if (found) { ++count; size += found->file->size; log_tag("dup:%s:%s:%s:%s:%" PRIu64 ": dup\n", disk->name, esc_tag(file->sub), found->disk->name, esc_tag(found->file->sub), found->file->size); printf("%12" PRIu64 " %s = %s\n", file->size, fmt_term(disk, file->sub), fmt_term(found->disk, found->file->sub)); hash_free(hash); } else { tommy_hashdyn_insert(&hashset, &hash->node, hash, hash_val); } } } tommy_hashdyn_foreach(&hashset, hash_free); tommy_hashdyn_done(&hashset); msg_status("\n"); msg_status("%8u duplicates, for %" PRIu64 " GB\n", count, size / GIGA); if (count) msg_status("There are duplicates!\n"); else msg_status("No duplicates\n"); log_tag("summary:dup_count:%u\n", count); log_tag("summary:dup_size:%" PRIu64 "\n", size); if (count == 0) { log_tag("summary:exit:unique\n"); } else { log_tag("summary:exit:dup\n"); } log_flush(); }