From e82cd8a19fc6e0bf35a28289b77e3502a12b8dc1 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 26 Dec 2025 11:51:48 -0700 Subject: [PATCH] Moving file groups in bulk to speed up tagging and moving files. --- wrolpi/files/lib.py | 72 ++++++++++++++++++++++++++++++++++++++++-- wrolpi/files/models.py | 16 ++++++---- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 882df4a4..9b09e385 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -1344,6 +1344,66 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi MOVE_CHUNK_SIZE = 100 +def _move_file_group_files(file_group: FileGroup, new_primary_path: pathlib.Path) -> None: + """Move all physical files of a FileGroup to a new directory. + + For directory moves where filenames are preserved, this moves each file from + the old directory to the new directory keeping the same filename. + + Args: + file_group: The FileGroup whose files should be moved + new_primary_path: The new path for the primary file + """ + new_directory = new_primary_path.parent + new_stem, _ = split_path_stem_and_suffix(new_primary_path, full=True) + + # Move each file in the FileGroup + for file_info in file_group.files: + relative_name = file_info['path'] + old_path = file_group.directory / relative_name + # Calculate new path: new_stem + original suffix + _, suffix = split_path_stem_and_suffix(relative_name) + new_path = pathlib.Path(f'{new_stem}{suffix}') + + if old_path.is_file(): + shutil.move(old_path, new_path) + logger.debug(f'Moved file: {old_path} -> {new_path}') + + +def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path, pathlib.Path]): + """Batch UPDATE FileGroups using raw SQL after physical files have been moved. + + Only updates directory and primary_path. Other fields (files, title, a_text) remain unchanged + because filenames are preserved in directory moves - only the parent directory changes. + + Args: + session: Database session + chunk_plan: Mapping of old_primary_path -> new_primary_path + """ + if not chunk_plan: + return + + # Use psycopg2 cursor for safe SQL building via mogrify + with get_db_curs() as curs: + # Build VALUES list for the update: (old_path, new_dir, new_path) + values = [ + (str(old_path), str(new_path.parent), str(new_path)) + for old_path, new_path in chunk_plan.items() + ] + values_sql = mogrify(curs, values) + + # Use UPDATE FROM VALUES pattern for efficient batch update + sql = f''' + UPDATE file_group AS fg SET + directory = v.new_dir, + primary_path = v.new_path, + indexed = CASE WHEN fg.data IS NULL THEN FALSE ELSE fg.indexed END + FROM (VALUES {values_sql}) AS v(old_path, new_dir, new_path) + WHERE fg.primary_path = v.old_path + ''' + curs.execute(sql) + + def add_files_to_refresh_count(count: int): from wrolpi.api_utils import api_app count = api_app.shared_ctx.refresh.get('counted_files', 0) + count @@ -1480,6 +1540,8 @@ async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.P # We ensured there are FileGroups while building the plan, maybe user deleted a file? raise RuntimeError('Could not get all FileGroups for move') + # Build chunk plan for bulk update + chunk_plan = {} for file_group_ in file_groups_: old_file = file_group_.primary_path new_primary_path_ = plan_[old_file] @@ -1487,7 +1549,9 @@ async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.P parent.mkdir(parents=True, exist_ok=True) if parent not in new_directories: new_directories.add(parent) - file_group_.move(new_primary_path_) + # Move physical files + _move_file_group_files(file_group_, new_primary_path_) + chunk_plan[old_file] = new_primary_path_ revert_plan[new_primary_path_] = old_file for directory_ in old_directories_: delete_directory(directory_) @@ -1498,8 +1562,10 @@ async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.P session.add_all([Directory(path=d, name=d.name) for d in missing_directories]) inserted_directories.update(missing_directories) - # Don't forget what has moved. - session.flush(file_groups_) + # Batch update all FileGroups in this chunk with raw SQL + _bulk_update_file_groups_db(session, chunk_plan) + # Flush Directory inserts + session.flush() with flags.refresh_discovery: try: diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index b520db9a..2f12dd8c 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -570,15 +570,20 @@ class FileGroup(ModelHelper, Base): logger.debug(f'Moved FileGroup: {self.primary_path} -> {new_primary_path}') + # Check if filename changed before updating primary_path + old_name = self.primary_path.name + filename_changed = old_name != new_primary_path.name + # Update the FileGroup with new directory and relative files self.files = new_files self.directory = new_directory self.primary_path = new_primary_path - # Update title if it was based on the filename - if self.title == self.primary_path.name: - self.title = new_primary_path.name - self.a_text = split_file_name_words(new_primary_path.name) + # Update title and a_text only if filename changed + if filename_changed: + if self.title == old_name: + self.title = new_primary_path.name + self.a_text = split_file_name_words(new_primary_path.name) # Note: `data` now contains relative paths that don't need updating! # The filenames stay the same, only the directory changes. @@ -586,8 +591,7 @@ class FileGroup(ModelHelper, Base): if not self.data: self.indexed = False - # Flush the changes to the FileGroup. - self.flush() + # Note: Caller is responsible for flushing/committing @property def location(self):