Raising NoPrimaryFile when no primary file can be found. Fixing bug where you could not delete some files if they shared some of their name.

This commit is contained in:
Roland 2023-09-15 12:03:09 -06:00
parent 92350b1d58
commit ac97bd7774
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
5 changed files with 66 additions and 30 deletions

View file

@ -13,7 +13,7 @@ import zipfile
from abc import ABC
from datetime import datetime
from itertools import zip_longest
from typing import List, Callable, Dict, Sequence
from typing import List, Callable, Dict, Sequence, Union
from typing import Tuple, Set
from unittest import mock
from unittest.mock import MagicMock
@ -375,7 +375,12 @@ def example_mobi(test_directory) -> pathlib.Path:
@pytest.fixture
def make_files_structure(test_directory) -> Callable:
def create_files(paths: List, file_groups: bool = False, session: Session = None) -> List[pathlib.Path]:
"""
A fixture which creates test files passed to it. If a list is provided, empty files will be created at those
locations. If a dict is provided, files will be created containing the value of the dict item.
"""
def create_files(paths: Union[List, Dict], file_groups: bool = False, session: Session = None) -> List[
pathlib.Path]:
files = []
@iterify(list)
@ -404,7 +409,7 @@ def make_files_structure(test_directory) -> Callable:
for idx, path in enumerate(files):
fg = FileGroup.from_paths(session, path)
files[idx] = fg
return files
return sorted(files) if isinstance(paths, dict) else files
return create_files

View file

@ -137,3 +137,9 @@ class InvalidConfig(APIError):
code = 'INVALID_CONFIG'
summary = 'Config is invalid'
status = HTTPStatus.BAD_REQUEST
class NoPrimaryFile(APIError):
code = 'NO_PRIMARY_FILE'
summary = 'Could not find a primary file in the provided group'
status = HTTPStatus.INTERNAL_SERVER_ERROR

View file

@ -25,7 +25,8 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk
from wrolpi.dates import now, from_timestamp
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
NoPrimaryFile
from wrolpi.events import Events
from wrolpi.files.models import FileGroup, Directory
from wrolpi.lang import ISO_639_CODES
@ -165,7 +166,9 @@ async def delete(*paths: Union[str, pathlib.Path]):
# Search for any FileGroups that have been tagged under this directory.
query = query.filter(FileGroup.primary_path.like(f'{path}/%'))
for (file_group, tag_file) in query:
raise FileGroupIsTagged(f"Cannot delete {file_group} because it is tagged")
if any(i for i in paths if i in file_group.my_paths()):
# File that will be deleted is in a Tagged FileGroup.
raise FileGroupIsTagged(f"Cannot delete {file_group} because it is tagged")
for path in paths:
if path.is_dir():
delete_directory(path, recursive=True)
@ -179,11 +182,6 @@ async def delete(*paths: Union[str, pathlib.Path]):
background_task(coro)
FILE_NAME_REGEX = re.compile(r'[_ .]')
FILE_BIN = which('file', '/usr/bin/file')
def _mimetype_suffix_map(path: Path, mimetype: str):
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
from wrolpi.files.ebooks import MOBI_MIMETYPE
@ -228,6 +226,9 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
return mimetype
FILE_BIN = which('file', '/usr/bin/file')
@functools.lru_cache(maxsize=10_000)
def get_mimetype(path: Path) -> str:
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
@ -299,8 +300,10 @@ def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
return files
def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> Union[pathlib.Path, List[pathlib.Path]]:
"""Given a list of files, return the file that we can model or index."""
def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> pathlib.Path:
"""Given a list of files, return the file that we can model or index.
@raise NoPrimaryFile: If not primary file can be found."""
from modules.archive.lib import is_singlefile_file
if len(files) == 0:
@ -332,10 +335,7 @@ def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> U
if mimetype.startswith('image/'):
return file
refresh_logger.debug(f'Cannot find primary file for group: {files}')
# Can't find a typical primary file.
return files
raise NoPrimaryFile(f'Cannot find primary file for group: {files}')
def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
@ -355,10 +355,10 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
values = dict()
for group in grouped:
# The primary file is the video/SingleFile/epub, etc.
primary_path_or_paths = get_primary_file(group)
if isinstance(primary_path_or_paths, pathlib.Path):
try:
primary_path = get_primary_file(group)
# Multiple files in this group.
primary_path: pathlib.Path = primary_path_or_paths
primary_path: pathlib.Path = primary_path
# The primary mimetype allows modelers to find its file_groups.
mimetype = get_mimetype(primary_path)
# The group uses a common modification_datetime so the group will be re-indexed when any of it's files
@ -368,9 +368,9 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
files = json.dumps(_paths_to_files_dict(group))
values[primary_path] = (modification_datetime, mimetype, size, files)
non_primary_files = {i for i in group if i != primary_path}
else:
except NoPrimaryFile:
# Cannot find primary path, create a `file_group` for each file.
for primary_path in primary_path_or_paths:
for primary_path in group:
primary_path: pathlib.Path
# The primary mimetype allows modelers to find its file_groups.
mimetype = get_mimetype(primary_path)

View file

@ -513,7 +513,7 @@ async def test_directory_crud(test_session, test_async_client, test_directory, a
@pytest.mark.asyncio
async def test_move(test_session, test_directory, make_files_structure, test_async_client):
foo, bar = make_files_structure({
baz, bar = make_files_structure({
'foo/bar.txt': 'bar',
'baz.txt': 'baz',
})
@ -529,14 +529,14 @@ async def test_move(test_session, test_directory, make_files_structure, test_asy
content = dict(paths=['foo', 'baz.txt'], destination='qux')
request, response = await test_async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not foo.exists()
assert not bar.exists()
assert (test_directory / 'qux/foo/bar.txt').is_file()
# mv foo/bar.txt qux
content = dict(paths=['qux/foo/bar.txt', ], destination='qux')
request, response = await test_async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not foo.exists()
assert not bar.exists()
assert (test_directory / 'qux/bar.txt').is_file()
assert (test_directory / 'qux/foo').is_dir()

View file

@ -14,7 +14,7 @@ from PIL import Image
from wrolpi.common import timer
from wrolpi.dates import now, from_timestamp
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
from wrolpi.files import lib, indexers
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile
@ -22,7 +22,7 @@ from wrolpi.vars import PROJECT_DIR
@pytest.mark.asyncio
async def test_delete(test_session, make_files_structure, test_directory):
async def test_delete_file(test_session, make_files_structure, test_directory):
"""
File in the media directory can be deleted.
"""
@ -54,7 +54,7 @@ async def test_delete(test_session, make_files_structure, test_directory):
@pytest.mark.asyncio
async def test_delete_multiple(test_session, make_files_structure, test_directory):
async def test_delete_file_multiple(test_session, make_files_structure, test_directory):
"""Multiple files can be deleted at once."""
foo, bar, baz = make_files_structure([
'archives/foo.txt',
@ -72,7 +72,27 @@ async def test_delete_multiple(test_session, make_files_structure, test_director
@pytest.mark.asyncio
async def test_delete_link(test_session, test_directory):
async def test_delete_file_names(test_session, make_files_structure, test_directory, tag_factory):
"""Will not refuse to delete a file that shares the name of a nearby file when they are in different FileGroups."""
foo, foo1 = make_files_structure({
'archives/foo': 'text',
'archives/foo1': 'text',
})
foo_fg = FileGroup.from_paths(test_session, foo)
foo1_fg = FileGroup.from_paths(test_session, foo1)
tag = tag_factory()
foo1_fg.add_tag(tag)
test_session.commit()
assert foo.is_file()
assert foo1.is_file()
await lib.delete('archives/foo')
assert not foo.is_file()
assert foo1.is_file()
@pytest.mark.asyncio
async def test_delete_file_link(test_session, test_directory):
"""Links can be deleted."""
foo, bar = test_directory / 'foo', test_directory / 'bar'
foo.touch()
@ -187,7 +207,7 @@ async def test_refresh_empty_media_directory(test_session, test_directory):
@pytest.mark.asyncio
async def test__upsert_files(test_session, make_files_structure, test_directory, assert_file_groups, video_file,
srt_file3):
bar, baz = make_files_structure({
baz, bar = make_files_structure({
'dir1/bar.txt': None,
'baz.txt': 'baz file',
})
@ -592,7 +612,7 @@ def test_group_files_by_stem(make_files_structure, test_directory):
def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, example_mobi, example_pdf,
singlefile_contents_factory):
singlefile_contents_factory, make_files_structure):
"""Test that the most important file is returned from a list of files."""
srt_file3 = srt_file3.rename(test_directory / f'{video_file.stem}.srt')
# Same primary_file no matter the order.
@ -612,6 +632,11 @@ def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, e
assert lib.get_primary_file([example_pdf, example_mobi]) == example_pdf
assert lib.get_primary_file([example_pdf, example_mobi, example_epub]) == example_epub
# No primary file in this group.
foo, bar = make_files_structure({'foo': 'text', 'bar': None})
with pytest.raises(NoPrimaryFile):
assert lib.get_primary_file([foo, bar])
def test_get_refresh_progress(test_client, test_session):
request, response = test_client.get('/api/files/refresh_progress')