Handling UTF file name error by renaming files to remove the bad characters.
Some checks are pending
CodeQL / Analyze (push) Waiting to run
Controller Tests / controller-tests (push) Waiting to run
Jest Tests / jest-tests (push) Waiting to run

This commit is contained in:
Roland 2025-12-21 13:24:37 -07:00
parent 457406a7f7
commit af15d59950
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
3 changed files with 144 additions and 2 deletions

View file

@ -4,6 +4,7 @@ import datetime
import functools
import glob
import json
import os
import pathlib
import re
import shutil
@ -65,6 +66,48 @@ def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
return names
def sanitize_filename_surrogates(path: pathlib.Path) -> pathlib.Path:
"""
Check if a path contains invalid UTF-8 surrogates. If so, rename the file
on disk to replace them with underscores.
Linux filesystems store filenames as bytes. Python represents invalid UTF-8
sequences as surrogates. These surrogates cannot be stored in PostgreSQL.
Returns the (possibly new) path.
"""
path_str = str(path)
# Check for surrogates by trying to encode as UTF-8
try:
path_str.encode('utf-8')
return path # Path is valid
except UnicodeEncodeError:
pass
# Get raw bytes from filesystem and decode, replacing invalid sequences
raw_bytes = os.fsencode(path)
# Replace invalid UTF-8 sequences with underscore
sanitized_name = raw_bytes.decode('utf-8', errors='replace').replace('\ufffd', '_')
sanitized_path = pathlib.Path(sanitized_name)
if path.exists() and sanitized_path != path:
if sanitized_path.exists():
# Target already exists, add a suffix
stem = sanitized_path.stem
suffix = sanitized_path.suffix
counter = 1
while sanitized_path.exists():
sanitized_path = sanitized_path.with_name(f'{stem}_{counter}{suffix}')
counter += 1
logger.warning(f'Renaming file with invalid UTF-8 characters: {path!r} -> {sanitized_path}')
path.rename(sanitized_path)
return sanitized_path
return path
def _get_file_dict(session: Session, file: pathlib.Path) -> Dict:
media_directory = get_media_directory()
try:
@ -712,7 +755,7 @@ async def apply_indexers():
while True:
# Continually query for Files that have not been indexed.
with get_db_session(commit=True) as session:
with get_db_session(commit=False) as session:
file_groups = session.query(FileGroup).filter(FileGroup.indexed != True).limit(20)
file_groups: List[FileGroup] = list(file_groups)
@ -732,6 +775,29 @@ async def apply_indexers():
# Sleep to catch cancel.
await asyncio.sleep(0)
# Commit may fail due to invalid UTF-8 surrogates in paths
try:
session.commit()
except UnicodeEncodeError as e:
session.rollback()
refresh_logger.warning(f'UnicodeEncodeError during indexer commit, attempting to fix: {e}')
# Find and fix the problematic file(s)
for fg in file_groups:
try:
sanitized = sanitize_filename_surrogates(fg.primary_path)
if sanitized != fg.primary_path:
fg.primary_path = sanitized
# Also update the files JSON
if fg.files:
fg.files = [
{**f, 'path': str(sanitize_filename_surrogates(pathlib.Path(f['path'])))}
for f in fg.files
]
except Exception as fix_error:
refresh_logger.error(f'Failed to sanitize path for FileGroup {fg.id}: {fix_error}')
# Retry commit after fixing
session.commit()
if file_group:
refresh_logger.debug(f'Indexed {processed} files near {file_group.primary_path}')

View file

@ -338,7 +338,10 @@ class FileGroup(ModelHelper, Base):
@classmethod
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
"""Create a new FileGroup which contains the provided file paths."""
from wrolpi.files.lib import get_primary_file, get_mimetype
from wrolpi.files.lib import get_primary_file, get_mimetype, sanitize_filename_surrogates
# Sanitize any paths with invalid UTF-8 characters (renames files on disk if needed)
paths = tuple(sanitize_filename_surrogates(p) for p in paths)
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
logger.trace(f'FileGroup.from_paths: {len(existing_groups)=}')

View file

@ -1340,3 +1340,76 @@ async def test_process_bulk_tag_job_remove_tags(async_client, test_session, make
fgs = test_session.query(FileGroup).all()
for fg in fgs:
assert 'to_remove' not in fg.tag_names
def test_sanitize_filename_surrogates_valid_path(test_directory, make_files_structure):
"""sanitize_filename_surrogates() returns the same path for valid UTF-8 filenames."""
foo, = make_files_structure({'foo.txt': 'content'})
result = lib.sanitize_filename_surrogates(foo)
assert result == foo
assert foo.exists()
def test_sanitize_filename_surrogates_with_emoji(test_directory, make_files_structure):
"""sanitize_filename_surrogates() handles valid emoji filenames correctly."""
# Valid emoji filename (this should work on most filesystems)
files = make_files_structure({'test_emoji_🎉.txt': 'content'})
emoji_file = files[0]
result = lib.sanitize_filename_surrogates(emoji_file)
assert result == emoji_file
assert emoji_file.exists()
def test_sanitize_filename_surrogates_nonexistent_path(test_directory):
"""sanitize_filename_surrogates() returns the path unchanged if file doesn't exist."""
nonexistent = test_directory / 'does_not_exist.txt'
result = lib.sanitize_filename_surrogates(nonexistent)
# Path doesn't exist, so it's returned unchanged (no rename attempted)
assert result == nonexistent
@pytest.mark.skipif(IS_MACOS, reason="macOS filesystem enforces UTF-8, can't create files with invalid surrogates")
def test_sanitize_filename_surrogates_renames_file(test_directory):
"""sanitize_filename_surrogates() renames files when it detects invalid UTF-8.
This test only runs on Linux where filesystems allow invalid UTF-8 filenames.
"""
import os
# Create a file with invalid UTF-8 bytes in the filename using raw bytes
# \xed\xa0\xbc is an invalid UTF-8 sequence (unpaired surrogate)
bad_filename_bytes = b'bad_\xed\xa0\xbc_file.txt'
bad_file_path_bytes = os.fsencode(test_directory) + b'/' + bad_filename_bytes
# Write content to the file using raw bytes path
with open(bad_file_path_bytes, 'wb') as f:
f.write(b'test content')
# Get the path as Python sees it (with surrogates)
bad_path = test_directory / os.fsdecode(bad_filename_bytes)
assert bad_path.exists()
# Verify the path string contains surrogates (can't be encoded to UTF-8)
with pytest.raises(UnicodeEncodeError):
str(bad_path).encode('utf-8')
# Call the sanitize function
result = lib.sanitize_filename_surrogates(bad_path)
# The file should have been renamed
assert result != bad_path
assert result.exists()
assert not bad_path.exists()
# Surrogates should be replaced with underscores - the result should be valid UTF-8
try:
str(result).encode('utf-8')
except UnicodeEncodeError:
pytest.fail("Result path still contains invalid UTF-8 surrogates")
# Content should be preserved
assert result.read_text() == 'test content'