Improve the pool command

The links are written only if they need to be updated.

HISTORY

@@ -1,6 +1,12 @@
 SnapRAID HISTORY
 ================
 
+11.2 2017/10
+============
+ * More efficient 'pool' command that updates only the links
+   that need to be updated. This ensures that no change is
+   done, avoiding to trigger a directory rescan of other programs.
+
 11.1 2017/05
 ============
  * Fixed the check command to correctly ignore errors on unused parity.

TODO

@@ -5,8 +5,6 @@ This is the list of TODO items for SnapRAID.
 
 - Next
 
-* Better "pool" command without deleting/creating unchanged links.
-
 * Fix dup command with symlinks. They are reported as dups.
 See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/57f99656/?limit=25#80c1
 

cmdline/pool.c

@@ -21,14 +21,55 @@
 #include "elem.h"
 #include "state.h"
 
+struct snapraid_pool {
+	char file[PATH_MAX];
+	char linkto[PATH_MAX];
+	int64_t mtime_sec;
+	int mtime_nsec;
+
+	/* nodes for data structures */
+	tommy_hashdyn_node node;
+};
+
+struct snapraid_pool* pool_alloc(const char* dir, const char* name, const char* linkto, const struct stat* st)
+{
+	struct snapraid_pool* pool;
+
+	pool = malloc_nofail(sizeof(struct snapraid_pool));
+	pathprint(pool->file, sizeof(pool->file), "%s%s", dir, name);
+	pathcpy(pool->linkto, sizeof(pool->linkto), linkto);
+	pool->mtime_sec = st->st_mtime;
+	pool->mtime_nsec = STAT_NSEC(st);
+
+	return pool;
+}
+
+static inline tommy_uint32_t pool_hash(const char* file)
+{
+	return tommy_hash_u32(0, file, strlen(file));
+}
+
+void pool_free(struct snapraid_pool* pool)
+{
+	free(pool);
+}
+
+int pool_compare(const void* void_arg, const void* void_data)
+{
+	const char* arg = void_arg;
+	const struct snapraid_pool* pool = void_data;
+
+	return strcmp(arg, pool->file);
+}
+
 /**
- * Clean a directory tree removing all the symlinks and empty directories.
+ * Remove empty dir.
  * Return == 0 if the directory is empty, and it can be removed
  */
-static int clean_dir(struct snapraid_state* state, const char* dir)
+static int clean_dir(const char* dir)
 {
 	DIR* d;
-	int ignored = 0;
+	int full = 0;
 
 	d = opendir(dir);
 	if (!d) {
@@ -88,21 +129,10 @@ static int clean_dir(struct snapraid_state* state, const char* dir)
 		}
 #endif
 
-		if (S_ISLNK(st.st_mode)) {
-			int ret;
-
-			/* delete the link */
-			ret = remove(path_next);
-			if (ret < 0) {
-				/* LCOV_EXCL_START */
-				log_fatal("Error removing symlink '%s'. %s.\n", path_next, strerror(errno));
-				exit(EXIT_FAILURE);
-				/* LCOV_EXCL_STOP */
-			}
-		} else if (S_ISDIR(st.st_mode)) {
+		if (S_ISDIR(st.st_mode)) {
 			/* recurse */
 			pathslash(path_next, sizeof(path_next));
-			if (clean_dir(state, path_next) == 0) {
+			if (clean_dir(path_next) == 0) {
 				int ret;
 
 				/* directory is empty, try to remove it */
@@ -114,7 +144,7 @@ static int clean_dir(struct snapraid_state* state, const char* dir)
 						/* because it could happen that the directory is in use */
 						/* and it cannot be removed */
 						log_fatal("Directory '%s' not removed because it's in use.\n", path_next);
-						ignored = 1;
+						full = 1;
 					} else
 #endif
 					{
@@ -125,12 +155,12 @@ static int clean_dir(struct snapraid_state* state, const char* dir)
 					}
 				}
 			} else {
-				/* something was ignored inside the subdir */
-				ignored = 1;
+				/* something is present */
+				full = 1;
 			}
 		} else {
-			ignored = 1;
-			msg_verbose("Ignoring pool file '%s'\n", path_next);
+			/* something is present */
+			full = 1;
 		}
 	}
 
@@ -141,17 +171,143 @@ static int clean_dir(struct snapraid_state* state, const char* dir)
 		/* LCOV_EXCL_STOP */
 	}
 
-	return ignored;
+	return full;
 }
 
 /**
- * Create a link to the specified disk entry.
+ * Read all the links in a directory tree.
  */
-static void make_link(const char* pool_dir, const char* share_dir, struct snapraid_disk* disk, const char* sub, int64_t mtime_sec, int mtime_nsec)
+static void read_dir(tommy_hashdyn* poolset, const char* base_dir, const char* sub_dir)
+{
+	char dir[PATH_MAX];
+	DIR* d;
+
+	pathprint(dir, sizeof(dir), "%s%s", base_dir, sub_dir);
+	d = opendir(dir);
+	if (!d) {
+		/* LCOV_EXCL_START */
+		log_fatal("Error opening pool directory '%s'. %s.\n", dir, strerror(errno));
+		exit(EXIT_FAILURE);
+		/* LCOV_EXCL_STOP */
+	}
+
+	while (1) {
+		char path_next[PATH_MAX];
+		struct stat st;
+		const char* name;
+		struct dirent* dd;
+
+		/* clear errno to detect erroneous conditions */
+		errno = 0;
+		dd = readdir(d);
+		if (dd == 0 && errno != 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error reading pool directory '%s'. %s.\n", dir, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+		if (dd == 0 && errno == 0) {
+			break; /* finished */
+		}
+
+		/* skip "." and ".." files */
+		name = dd->d_name;
+		if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
+			continue;
+
+		pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
+
+#if HAVE_STRUCT_DIRENT_D_STAT
+		/* convert dirent to lstat result */
+		dirent_lstat(dd, &st);
+
+		/* if the st_mode field is missing, takes care to fill it using normal lstat() */
+		/* at now this can happen only in Windows */
+		if (st.st_mode == 0) {
+			if (lstat(path_next, &st) != 0) {
+				/* LCOV_EXCL_START */
+				log_fatal("Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
+				exit(EXIT_FAILURE);
+				/* LCOV_EXCL_STOP */
+			}
+		}
+#else
+		/* get lstat info about the file */
+		if (lstat(path_next, &st) != 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+#endif
+
+		if (S_ISLNK(st.st_mode)) {
+			struct snapraid_pool* pool;
+			char linkto[PATH_MAX];
+			int ret;
+
+			ret = readlink(path_next, linkto, sizeof(linkto));
+			if (ret < 0 || ret == sizeof(linkto)) {
+				/* LCOV_EXCL_START */
+				log_fatal("Error in readlink symlink '%s'. %s.\n", path_next, strerror(errno));
+				exit(EXIT_FAILURE);
+				/* LCOV_EXCL_STOP */
+			}
+			linkto[ret] = 0;
+
+			/* store the link info */
+			pool = pool_alloc(sub_dir, name, linkto, &st);
+
+			tommy_hashdyn_insert(poolset, &pool->node, pool, pool_hash(pool->file));
+
+		} else if (S_ISDIR(st.st_mode)) {
+			pathprint(path_next, sizeof(path_next), "%s%s/", sub_dir, name);
+
+			read_dir(poolset, base_dir, path_next);
+		} else {
+			msg_verbose("Ignoring pool file '%s'\n", path_next);
+		}
+	}
+
+	if (closedir(d) != 0) {
+		/* LCOV_EXCL_START */
+		log_fatal("Error closing pool directory '%s'. %s.\n", dir, strerror(errno));
+		exit(EXIT_FAILURE);
+		/* LCOV_EXCL_STOP */
+	}
+}
+
+/**
+ * Remove the link
+ */
+static void remove_link(void* void_arg, void* void_pool)
+{
+	char path[PATH_MAX];
+	const char* arg = void_arg;
+	struct snapraid_pool* pool = void_pool;
+	int ret;
+
+	pathprint(path, sizeof(path), "%s%s", arg, pool->file);
+
+	/* delete the link */
+	ret = remove(path);
+	if (ret < 0) {
+		/* LCOV_EXCL_START */
+		log_fatal("Error removing symlink '%s'. %s.\n", path, strerror(errno));
+		exit(EXIT_FAILURE);
+		/* LCOV_EXCL_STOP */
+	}
+}
+
+/**
+ * Create a link to the specified disk link.
+ */
+static void make_link(tommy_hashdyn* poolset, const char* pool_dir, const char* share_dir, struct snapraid_disk* disk, const char* sub, int64_t mtime_sec, int mtime_nsec)
 {
 	char path[PATH_MAX];
 	char linkto[PATH_MAX];
 	char linkto_exported[PATH_MAX];
+	struct snapraid_pool* found;
 	int ret;
 
 	/* make the source path */
@@ -166,6 +322,34 @@ static void make_link(const char* pool_dir, const char* share_dir, struct snapra
 		pathprint(linkto, sizeof(linkto), "%s%s", disk->dir, sub);
 	}
 
+	/* search for the sub path */
+	found = tommy_hashdyn_search(poolset, pool_compare, sub, pool_hash(sub));
+	if (found) {
+		/* remove from the set */
+		tommy_hashdyn_remove_existing(poolset, &found->node);
+
+		/* check if the info match */
+		if (found->mtime_sec == mtime_sec
+			&& found->mtime_nsec == mtime_nsec
+			&& strcmp(found->linkto, linkto) == 0
+		) {
+			/* nothing to do */
+			pool_free(found);
+			return;
+		}
+
+		/* delete the link */
+		ret = remove(path);
+		if (ret < 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error removing symlink '%s'. %s.\n", path, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+
+		pool_free(found);
+	}
+
 	/* create the ancestor directories */
 	ret = mkancestor(path);
 	if (ret != 0) {
@@ -210,11 +394,14 @@ static void make_link(const char* pool_dir, const char* share_dir, struct snapra
 
 void state_pool(struct snapraid_state* state)
 {
+	tommy_hashdyn poolset;
 	tommy_node* i;
 	char pool_dir[PATH_MAX];
 	char share_dir[PATH_MAX];
 	unsigned count;
 
+	tommy_hashdyn_init(&poolset);
+
 	if (state->pool[0] == 0) {
 		/* LCOV_EXCL_START */
 		log_fatal("To use the 'pool' command you must set the pool directory in the configuration file\n");
@@ -222,7 +409,7 @@ void state_pool(struct snapraid_state* state)
 		/* LCOV_EXCL_STOP */
 	}
 
-	msg_progress("Cleaning...\n");
+	msg_progress("Reading...\n");
 
 	/* pool directory with final slash */
 	pathprint(pool_dir, sizeof(pool_dir), "%s", state->pool);
@@ -232,10 +419,10 @@ void state_pool(struct snapraid_state* state)
 	pathprint(share_dir, sizeof(share_dir), "%s", state->share);
 	pathslash(share_dir, sizeof(share_dir));
 
-	/* first clear the previous pool tree */
-	clean_dir(state, pool_dir);
+	/* first read the previous pool tree */
+	read_dir(&poolset, pool_dir, "");
 
-	msg_progress("Pooling...\n");
+	msg_progress("Writing...\n");
 
 	/* for each disk */
 	count = 0;
@@ -246,24 +433,35 @@ void state_pool(struct snapraid_state* state)
 		/* for each file */
 		for (j = disk->filelist; j != 0; j = j->next) {
 			struct snapraid_file* file = j->data;
-			make_link(pool_dir, share_dir, disk, file->sub, file->mtime_sec, file->mtime_nsec);
+			make_link(&poolset, pool_dir, share_dir, disk, file->sub, file->mtime_sec, file->mtime_nsec);
 			++count;
 		}
 
 		/* for each link */
 		for (j = disk->linklist; j != 0; j = j->next) {
 			struct snapraid_link* slink = j->data;
-			make_link(pool_dir, share_dir, disk, slink->sub, 0, 0);
+			make_link(&poolset, pool_dir, share_dir, disk, slink->sub, 0, 0);
 			++count;
 		}
 
 		/* we ignore empty dirs in disk->dir */
 	}
 
+	msg_progress("Cleaning...\n");
+
+	/* delete all the remaining links */
+	tommy_hashdyn_foreach_arg(&poolset, (tommy_foreach_arg_func*)remove_link, pool_dir);
+
+	/* delete empty dirs */
+	clean_dir(pool_dir);
+
+	tommy_hashdyn_foreach(&poolset, (tommy_foreach_func*)pool_free);
+	tommy_hashdyn_done(&poolset);
+
 	if (count)
-		msg_status("%u links created\n", count);
+		msg_status("%u links\n", count);
 	else
-		msg_status("No link created\n");
+		msg_status("No link\n");
 
 	log_tag("summary:link_count::%u\n", count);
 	log_tag("summary:exit:ok\n");