mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
Handling UTF file name error by renaming files to remove the bad characters.
This commit is contained in:
parent
457406a7f7
commit
af15d59950
3 changed files with 144 additions and 2 deletions
|
|
@ -4,6 +4,7 @@ import datetime
|
|||
import functools
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
|
|
@ -65,6 +66,48 @@ def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
|
|||
return names
|
||||
|
||||
|
||||
def sanitize_filename_surrogates(path: pathlib.Path) -> pathlib.Path:
|
||||
"""
|
||||
Check if a path contains invalid UTF-8 surrogates. If so, rename the file
|
||||
on disk to replace them with underscores.
|
||||
|
||||
Linux filesystems store filenames as bytes. Python represents invalid UTF-8
|
||||
sequences as surrogates. These surrogates cannot be stored in PostgreSQL.
|
||||
|
||||
Returns the (possibly new) path.
|
||||
"""
|
||||
path_str = str(path)
|
||||
|
||||
# Check for surrogates by trying to encode as UTF-8
|
||||
try:
|
||||
path_str.encode('utf-8')
|
||||
return path # Path is valid
|
||||
except UnicodeEncodeError:
|
||||
pass
|
||||
|
||||
# Get raw bytes from filesystem and decode, replacing invalid sequences
|
||||
raw_bytes = os.fsencode(path)
|
||||
# Replace invalid UTF-8 sequences with underscore
|
||||
sanitized_name = raw_bytes.decode('utf-8', errors='replace').replace('\ufffd', '_')
|
||||
sanitized_path = pathlib.Path(sanitized_name)
|
||||
|
||||
if path.exists() and sanitized_path != path:
|
||||
if sanitized_path.exists():
|
||||
# Target already exists, add a suffix
|
||||
stem = sanitized_path.stem
|
||||
suffix = sanitized_path.suffix
|
||||
counter = 1
|
||||
while sanitized_path.exists():
|
||||
sanitized_path = sanitized_path.with_name(f'{stem}_{counter}{suffix}')
|
||||
counter += 1
|
||||
|
||||
logger.warning(f'Renaming file with invalid UTF-8 characters: {path!r} -> {sanitized_path}')
|
||||
path.rename(sanitized_path)
|
||||
return sanitized_path
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def _get_file_dict(session: Session, file: pathlib.Path) -> Dict:
|
||||
media_directory = get_media_directory()
|
||||
try:
|
||||
|
|
@ -712,7 +755,7 @@ async def apply_indexers():
|
|||
|
||||
while True:
|
||||
# Continually query for Files that have not been indexed.
|
||||
with get_db_session(commit=True) as session:
|
||||
with get_db_session(commit=False) as session:
|
||||
file_groups = session.query(FileGroup).filter(FileGroup.indexed != True).limit(20)
|
||||
file_groups: List[FileGroup] = list(file_groups)
|
||||
|
||||
|
|
@ -732,6 +775,29 @@ async def apply_indexers():
|
|||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# Commit may fail due to invalid UTF-8 surrogates in paths
|
||||
try:
|
||||
session.commit()
|
||||
except UnicodeEncodeError as e:
|
||||
session.rollback()
|
||||
refresh_logger.warning(f'UnicodeEncodeError during indexer commit, attempting to fix: {e}')
|
||||
# Find and fix the problematic file(s)
|
||||
for fg in file_groups:
|
||||
try:
|
||||
sanitized = sanitize_filename_surrogates(fg.primary_path)
|
||||
if sanitized != fg.primary_path:
|
||||
fg.primary_path = sanitized
|
||||
# Also update the files JSON
|
||||
if fg.files:
|
||||
fg.files = [
|
||||
{**f, 'path': str(sanitize_filename_surrogates(pathlib.Path(f['path'])))}
|
||||
for f in fg.files
|
||||
]
|
||||
except Exception as fix_error:
|
||||
refresh_logger.error(f'Failed to sanitize path for FileGroup {fg.id}: {fix_error}')
|
||||
# Retry commit after fixing
|
||||
session.commit()
|
||||
|
||||
if file_group:
|
||||
refresh_logger.debug(f'Indexed {processed} files near {file_group.primary_path}')
|
||||
|
||||
|
|
|
|||
|
|
@ -338,7 +338,10 @@ class FileGroup(ModelHelper, Base):
|
|||
@classmethod
|
||||
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
|
||||
"""Create a new FileGroup which contains the provided file paths."""
|
||||
from wrolpi.files.lib import get_primary_file, get_mimetype
|
||||
from wrolpi.files.lib import get_primary_file, get_mimetype, sanitize_filename_surrogates
|
||||
|
||||
# Sanitize any paths with invalid UTF-8 characters (renames files on disk if needed)
|
||||
paths = tuple(sanitize_filename_surrogates(p) for p in paths)
|
||||
|
||||
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
|
||||
logger.trace(f'FileGroup.from_paths: {len(existing_groups)=}')
|
||||
|
|
|
|||
|
|
@ -1340,3 +1340,76 @@ async def test_process_bulk_tag_job_remove_tags(async_client, test_session, make
|
|||
fgs = test_session.query(FileGroup).all()
|
||||
for fg in fgs:
|
||||
assert 'to_remove' not in fg.tag_names
|
||||
|
||||
|
||||
def test_sanitize_filename_surrogates_valid_path(test_directory, make_files_structure):
|
||||
"""sanitize_filename_surrogates() returns the same path for valid UTF-8 filenames."""
|
||||
foo, = make_files_structure({'foo.txt': 'content'})
|
||||
|
||||
result = lib.sanitize_filename_surrogates(foo)
|
||||
|
||||
assert result == foo
|
||||
assert foo.exists()
|
||||
|
||||
|
||||
def test_sanitize_filename_surrogates_with_emoji(test_directory, make_files_structure):
|
||||
"""sanitize_filename_surrogates() handles valid emoji filenames correctly."""
|
||||
# Valid emoji filename (this should work on most filesystems)
|
||||
files = make_files_structure({'test_emoji_🎉.txt': 'content'})
|
||||
emoji_file = files[0]
|
||||
|
||||
result = lib.sanitize_filename_surrogates(emoji_file)
|
||||
|
||||
assert result == emoji_file
|
||||
assert emoji_file.exists()
|
||||
|
||||
|
||||
def test_sanitize_filename_surrogates_nonexistent_path(test_directory):
|
||||
"""sanitize_filename_surrogates() returns the path unchanged if file doesn't exist."""
|
||||
nonexistent = test_directory / 'does_not_exist.txt'
|
||||
|
||||
result = lib.sanitize_filename_surrogates(nonexistent)
|
||||
|
||||
# Path doesn't exist, so it's returned unchanged (no rename attempted)
|
||||
assert result == nonexistent
|
||||
|
||||
|
||||
@pytest.mark.skipif(IS_MACOS, reason="macOS filesystem enforces UTF-8, can't create files with invalid surrogates")
|
||||
def test_sanitize_filename_surrogates_renames_file(test_directory):
|
||||
"""sanitize_filename_surrogates() renames files when it detects invalid UTF-8.
|
||||
|
||||
This test only runs on Linux where filesystems allow invalid UTF-8 filenames.
|
||||
"""
|
||||
import os
|
||||
|
||||
# Create a file with invalid UTF-8 bytes in the filename using raw bytes
|
||||
# \xed\xa0\xbc is an invalid UTF-8 sequence (unpaired surrogate)
|
||||
bad_filename_bytes = b'bad_\xed\xa0\xbc_file.txt'
|
||||
bad_file_path_bytes = os.fsencode(test_directory) + b'/' + bad_filename_bytes
|
||||
|
||||
# Write content to the file using raw bytes path
|
||||
with open(bad_file_path_bytes, 'wb') as f:
|
||||
f.write(b'test content')
|
||||
|
||||
# Get the path as Python sees it (with surrogates)
|
||||
bad_path = test_directory / os.fsdecode(bad_filename_bytes)
|
||||
assert bad_path.exists()
|
||||
|
||||
# Verify the path string contains surrogates (can't be encoded to UTF-8)
|
||||
with pytest.raises(UnicodeEncodeError):
|
||||
str(bad_path).encode('utf-8')
|
||||
|
||||
# Call the sanitize function
|
||||
result = lib.sanitize_filename_surrogates(bad_path)
|
||||
|
||||
# The file should have been renamed
|
||||
assert result != bad_path
|
||||
assert result.exists()
|
||||
assert not bad_path.exists()
|
||||
# Surrogates should be replaced with underscores - the result should be valid UTF-8
|
||||
try:
|
||||
str(result).encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
pytest.fail("Result path still contains invalid UTF-8 surrogates")
|
||||
# Content should be preserved
|
||||
assert result.read_text() == 'test content'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue