From af15d599505554e651f724be5df18f782c8232e7 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 21 Dec 2025 13:24:37 -0700 Subject: [PATCH] Handling UTF file name error by renaming files to remove the bad characters. --- wrolpi/files/lib.py | 68 +++++++++++++++++++++++++++++++- wrolpi/files/models.py | 5 ++- wrolpi/files/test/test_lib.py | 73 +++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 2 deletions(-) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 8e284b10..8331f587 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -4,6 +4,7 @@ import datetime import functools import glob import json +import os import pathlib import re import shutil @@ -65,6 +66,48 @@ def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]: return names +def sanitize_filename_surrogates(path: pathlib.Path) -> pathlib.Path: + """ + Check if a path contains invalid UTF-8 surrogates. If so, rename the file + on disk to replace them with underscores. + + Linux filesystems store filenames as bytes. Python represents invalid UTF-8 + sequences as surrogates. These surrogates cannot be stored in PostgreSQL. + + Returns the (possibly new) path. + """ + path_str = str(path) + + # Check for surrogates by trying to encode as UTF-8 + try: + path_str.encode('utf-8') + return path # Path is valid + except UnicodeEncodeError: + pass + + # Get raw bytes from filesystem and decode, replacing invalid sequences + raw_bytes = os.fsencode(path) + # Replace invalid UTF-8 sequences with underscore + sanitized_name = raw_bytes.decode('utf-8', errors='replace').replace('\ufffd', '_') + sanitized_path = pathlib.Path(sanitized_name) + + if path.exists() and sanitized_path != path: + if sanitized_path.exists(): + # Target already exists, add a suffix + stem = sanitized_path.stem + suffix = sanitized_path.suffix + counter = 1 + while sanitized_path.exists(): + sanitized_path = sanitized_path.with_name(f'{stem}_{counter}{suffix}') + counter += 1 + + logger.warning(f'Renaming file with invalid UTF-8 characters: {path!r} -> {sanitized_path}') + path.rename(sanitized_path) + return sanitized_path + + return path + + def _get_file_dict(session: Session, file: pathlib.Path) -> Dict: media_directory = get_media_directory() try: @@ -712,7 +755,7 @@ async def apply_indexers(): while True: # Continually query for Files that have not been indexed. - with get_db_session(commit=True) as session: + with get_db_session(commit=False) as session: file_groups = session.query(FileGroup).filter(FileGroup.indexed != True).limit(20) file_groups: List[FileGroup] = list(file_groups) @@ -732,6 +775,29 @@ async def apply_indexers(): # Sleep to catch cancel. await asyncio.sleep(0) + # Commit may fail due to invalid UTF-8 surrogates in paths + try: + session.commit() + except UnicodeEncodeError as e: + session.rollback() + refresh_logger.warning(f'UnicodeEncodeError during indexer commit, attempting to fix: {e}') + # Find and fix the problematic file(s) + for fg in file_groups: + try: + sanitized = sanitize_filename_surrogates(fg.primary_path) + if sanitized != fg.primary_path: + fg.primary_path = sanitized + # Also update the files JSON + if fg.files: + fg.files = [ + {**f, 'path': str(sanitize_filename_surrogates(pathlib.Path(f['path'])))} + for f in fg.files + ] + except Exception as fix_error: + refresh_logger.error(f'Failed to sanitize path for FileGroup {fg.id}: {fix_error}') + # Retry commit after fixing + session.commit() + if file_group: refresh_logger.debug(f'Indexed {processed} files near {file_group.primary_path}') diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 3379bbca..352e327a 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -338,7 +338,10 @@ class FileGroup(ModelHelper, Base): @classmethod def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup': """Create a new FileGroup which contains the provided file paths.""" - from wrolpi.files.lib import get_primary_file, get_mimetype + from wrolpi.files.lib import get_primary_file, get_mimetype, sanitize_filename_surrogates + + # Sanitize any paths with invalid UTF-8 characters (renames files on disk if needed) + paths = tuple(sanitize_filename_surrogates(p) for p in paths) existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all() logger.trace(f'FileGroup.from_paths: {len(existing_groups)=}') diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index e7d662ef..ad3d6b1a 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -1340,3 +1340,76 @@ async def test_process_bulk_tag_job_remove_tags(async_client, test_session, make fgs = test_session.query(FileGroup).all() for fg in fgs: assert 'to_remove' not in fg.tag_names + + +def test_sanitize_filename_surrogates_valid_path(test_directory, make_files_structure): + """sanitize_filename_surrogates() returns the same path for valid UTF-8 filenames.""" + foo, = make_files_structure({'foo.txt': 'content'}) + + result = lib.sanitize_filename_surrogates(foo) + + assert result == foo + assert foo.exists() + + +def test_sanitize_filename_surrogates_with_emoji(test_directory, make_files_structure): + """sanitize_filename_surrogates() handles valid emoji filenames correctly.""" + # Valid emoji filename (this should work on most filesystems) + files = make_files_structure({'test_emoji_🎉.txt': 'content'}) + emoji_file = files[0] + + result = lib.sanitize_filename_surrogates(emoji_file) + + assert result == emoji_file + assert emoji_file.exists() + + +def test_sanitize_filename_surrogates_nonexistent_path(test_directory): + """sanitize_filename_surrogates() returns the path unchanged if file doesn't exist.""" + nonexistent = test_directory / 'does_not_exist.txt' + + result = lib.sanitize_filename_surrogates(nonexistent) + + # Path doesn't exist, so it's returned unchanged (no rename attempted) + assert result == nonexistent + + +@pytest.mark.skipif(IS_MACOS, reason="macOS filesystem enforces UTF-8, can't create files with invalid surrogates") +def test_sanitize_filename_surrogates_renames_file(test_directory): + """sanitize_filename_surrogates() renames files when it detects invalid UTF-8. + + This test only runs on Linux where filesystems allow invalid UTF-8 filenames. + """ + import os + + # Create a file with invalid UTF-8 bytes in the filename using raw bytes + # \xed\xa0\xbc is an invalid UTF-8 sequence (unpaired surrogate) + bad_filename_bytes = b'bad_\xed\xa0\xbc_file.txt' + bad_file_path_bytes = os.fsencode(test_directory) + b'/' + bad_filename_bytes + + # Write content to the file using raw bytes path + with open(bad_file_path_bytes, 'wb') as f: + f.write(b'test content') + + # Get the path as Python sees it (with surrogates) + bad_path = test_directory / os.fsdecode(bad_filename_bytes) + assert bad_path.exists() + + # Verify the path string contains surrogates (can't be encoded to UTF-8) + with pytest.raises(UnicodeEncodeError): + str(bad_path).encode('utf-8') + + # Call the sanitize function + result = lib.sanitize_filename_surrogates(bad_path) + + # The file should have been renamed + assert result != bad_path + assert result.exists() + assert not bad_path.exists() + # Surrogates should be replaced with underscores - the result should be valid UTF-8 + try: + str(result).encode('utf-8') + except UnicodeEncodeError: + pytest.fail("Result path still contains invalid UTF-8 surrogates") + # Content should be preserved + assert result.read_text() == 'test content'