mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
Raising NoPrimaryFile when no primary file can be found. Fixing bug where you could not delete some files if they shared some of their name.
This commit is contained in:
parent
92350b1d58
commit
ac97bd7774
5 changed files with 66 additions and 30 deletions
|
|
@ -13,7 +13,7 @@ import zipfile
|
|||
from abc import ABC
|
||||
from datetime import datetime
|
||||
from itertools import zip_longest
|
||||
from typing import List, Callable, Dict, Sequence
|
||||
from typing import List, Callable, Dict, Sequence, Union
|
||||
from typing import Tuple, Set
|
||||
from unittest import mock
|
||||
from unittest.mock import MagicMock
|
||||
|
|
@ -375,7 +375,12 @@ def example_mobi(test_directory) -> pathlib.Path:
|
|||
|
||||
@pytest.fixture
|
||||
def make_files_structure(test_directory) -> Callable:
|
||||
def create_files(paths: List, file_groups: bool = False, session: Session = None) -> List[pathlib.Path]:
|
||||
"""
|
||||
A fixture which creates test files passed to it. If a list is provided, empty files will be created at those
|
||||
locations. If a dict is provided, files will be created containing the value of the dict item.
|
||||
"""
|
||||
def create_files(paths: Union[List, Dict], file_groups: bool = False, session: Session = None) -> List[
|
||||
pathlib.Path]:
|
||||
files = []
|
||||
|
||||
@iterify(list)
|
||||
|
|
@ -404,7 +409,7 @@ def make_files_structure(test_directory) -> Callable:
|
|||
for idx, path in enumerate(files):
|
||||
fg = FileGroup.from_paths(session, path)
|
||||
files[idx] = fg
|
||||
return files
|
||||
return sorted(files) if isinstance(paths, dict) else files
|
||||
|
||||
return create_files
|
||||
|
||||
|
|
|
|||
|
|
@ -137,3 +137,9 @@ class InvalidConfig(APIError):
|
|||
code = 'INVALID_CONFIG'
|
||||
summary = 'Config is invalid'
|
||||
status = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class NoPrimaryFile(APIError):
|
||||
code = 'NO_PRIMARY_FILE'
|
||||
summary = 'Could not find a primary file in the provided group'
|
||||
status = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
|
|
|
|||
|
|
@ -25,7 +25,8 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
|
|||
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk
|
||||
from wrolpi.dates import now, from_timestamp
|
||||
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
|
||||
NoPrimaryFile
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.models import FileGroup, Directory
|
||||
from wrolpi.lang import ISO_639_CODES
|
||||
|
|
@ -165,7 +166,9 @@ async def delete(*paths: Union[str, pathlib.Path]):
|
|||
# Search for any FileGroups that have been tagged under this directory.
|
||||
query = query.filter(FileGroup.primary_path.like(f'{path}/%'))
|
||||
for (file_group, tag_file) in query:
|
||||
raise FileGroupIsTagged(f"Cannot delete {file_group} because it is tagged")
|
||||
if any(i for i in paths if i in file_group.my_paths()):
|
||||
# File that will be deleted is in a Tagged FileGroup.
|
||||
raise FileGroupIsTagged(f"Cannot delete {file_group} because it is tagged")
|
||||
for path in paths:
|
||||
if path.is_dir():
|
||||
delete_directory(path, recursive=True)
|
||||
|
|
@ -179,11 +182,6 @@ async def delete(*paths: Union[str, pathlib.Path]):
|
|||
background_task(coro)
|
||||
|
||||
|
||||
FILE_NAME_REGEX = re.compile(r'[_ .]')
|
||||
|
||||
FILE_BIN = which('file', '/usr/bin/file')
|
||||
|
||||
|
||||
def _mimetype_suffix_map(path: Path, mimetype: str):
|
||||
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
|
||||
from wrolpi.files.ebooks import MOBI_MIMETYPE
|
||||
|
|
@ -228,6 +226,9 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
|
|||
return mimetype
|
||||
|
||||
|
||||
FILE_BIN = which('file', '/usr/bin/file')
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=10_000)
|
||||
def get_mimetype(path: Path) -> str:
|
||||
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
|
||||
|
|
@ -299,8 +300,10 @@ def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
|
|||
return files
|
||||
|
||||
|
||||
def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> Union[pathlib.Path, List[pathlib.Path]]:
|
||||
"""Given a list of files, return the file that we can model or index."""
|
||||
def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> pathlib.Path:
|
||||
"""Given a list of files, return the file that we can model or index.
|
||||
|
||||
@raise NoPrimaryFile: If not primary file can be found."""
|
||||
from modules.archive.lib import is_singlefile_file
|
||||
|
||||
if len(files) == 0:
|
||||
|
|
@ -332,10 +335,7 @@ def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> U
|
|||
if mimetype.startswith('image/'):
|
||||
return file
|
||||
|
||||
refresh_logger.debug(f'Cannot find primary file for group: {files}')
|
||||
|
||||
# Can't find a typical primary file.
|
||||
return files
|
||||
raise NoPrimaryFile(f'Cannot find primary file for group: {files}')
|
||||
|
||||
|
||||
def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
|
||||
|
|
@ -355,10 +355,10 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
|
|||
values = dict()
|
||||
for group in grouped:
|
||||
# The primary file is the video/SingleFile/epub, etc.
|
||||
primary_path_or_paths = get_primary_file(group)
|
||||
if isinstance(primary_path_or_paths, pathlib.Path):
|
||||
try:
|
||||
primary_path = get_primary_file(group)
|
||||
# Multiple files in this group.
|
||||
primary_path: pathlib.Path = primary_path_or_paths
|
||||
primary_path: pathlib.Path = primary_path
|
||||
# The primary mimetype allows modelers to find its file_groups.
|
||||
mimetype = get_mimetype(primary_path)
|
||||
# The group uses a common modification_datetime so the group will be re-indexed when any of it's files
|
||||
|
|
@ -368,9 +368,9 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
|
|||
files = json.dumps(_paths_to_files_dict(group))
|
||||
values[primary_path] = (modification_datetime, mimetype, size, files)
|
||||
non_primary_files = {i for i in group if i != primary_path}
|
||||
else:
|
||||
except NoPrimaryFile:
|
||||
# Cannot find primary path, create a `file_group` for each file.
|
||||
for primary_path in primary_path_or_paths:
|
||||
for primary_path in group:
|
||||
primary_path: pathlib.Path
|
||||
# The primary mimetype allows modelers to find its file_groups.
|
||||
mimetype = get_mimetype(primary_path)
|
||||
|
|
|
|||
|
|
@ -513,7 +513,7 @@ async def test_directory_crud(test_session, test_async_client, test_directory, a
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move(test_session, test_directory, make_files_structure, test_async_client):
|
||||
foo, bar = make_files_structure({
|
||||
baz, bar = make_files_structure({
|
||||
'foo/bar.txt': 'bar',
|
||||
'baz.txt': 'baz',
|
||||
})
|
||||
|
|
@ -529,14 +529,14 @@ async def test_move(test_session, test_directory, make_files_structure, test_asy
|
|||
content = dict(paths=['foo', 'baz.txt'], destination='qux')
|
||||
request, response = await test_async_client.post('/api/files/move', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert not foo.exists()
|
||||
assert not bar.exists()
|
||||
assert (test_directory / 'qux/foo/bar.txt').is_file()
|
||||
|
||||
# mv foo/bar.txt qux
|
||||
content = dict(paths=['qux/foo/bar.txt', ], destination='qux')
|
||||
request, response = await test_async_client.post('/api/files/move', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert not foo.exists()
|
||||
assert not bar.exists()
|
||||
assert (test_directory / 'qux/bar.txt').is_file()
|
||||
assert (test_directory / 'qux/foo').is_dir()
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from PIL import Image
|
|||
|
||||
from wrolpi.common import timer
|
||||
from wrolpi.dates import now, from_timestamp
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
|
||||
from wrolpi.files import lib, indexers
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import TagFile
|
||||
|
|
@ -22,7 +22,7 @@ from wrolpi.vars import PROJECT_DIR
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete(test_session, make_files_structure, test_directory):
|
||||
async def test_delete_file(test_session, make_files_structure, test_directory):
|
||||
"""
|
||||
File in the media directory can be deleted.
|
||||
"""
|
||||
|
|
@ -54,7 +54,7 @@ async def test_delete(test_session, make_files_structure, test_directory):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_multiple(test_session, make_files_structure, test_directory):
|
||||
async def test_delete_file_multiple(test_session, make_files_structure, test_directory):
|
||||
"""Multiple files can be deleted at once."""
|
||||
foo, bar, baz = make_files_structure([
|
||||
'archives/foo.txt',
|
||||
|
|
@ -72,7 +72,27 @@ async def test_delete_multiple(test_session, make_files_structure, test_director
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_link(test_session, test_directory):
|
||||
async def test_delete_file_names(test_session, make_files_structure, test_directory, tag_factory):
|
||||
"""Will not refuse to delete a file that shares the name of a nearby file when they are in different FileGroups."""
|
||||
foo, foo1 = make_files_structure({
|
||||
'archives/foo': 'text',
|
||||
'archives/foo1': 'text',
|
||||
})
|
||||
foo_fg = FileGroup.from_paths(test_session, foo)
|
||||
foo1_fg = FileGroup.from_paths(test_session, foo1)
|
||||
tag = tag_factory()
|
||||
foo1_fg.add_tag(tag)
|
||||
test_session.commit()
|
||||
assert foo.is_file()
|
||||
assert foo1.is_file()
|
||||
|
||||
await lib.delete('archives/foo')
|
||||
assert not foo.is_file()
|
||||
assert foo1.is_file()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_file_link(test_session, test_directory):
|
||||
"""Links can be deleted."""
|
||||
foo, bar = test_directory / 'foo', test_directory / 'bar'
|
||||
foo.touch()
|
||||
|
|
@ -187,7 +207,7 @@ async def test_refresh_empty_media_directory(test_session, test_directory):
|
|||
@pytest.mark.asyncio
|
||||
async def test__upsert_files(test_session, make_files_structure, test_directory, assert_file_groups, video_file,
|
||||
srt_file3):
|
||||
bar, baz = make_files_structure({
|
||||
baz, bar = make_files_structure({
|
||||
'dir1/bar.txt': None,
|
||||
'baz.txt': 'baz file',
|
||||
})
|
||||
|
|
@ -592,7 +612,7 @@ def test_group_files_by_stem(make_files_structure, test_directory):
|
|||
|
||||
|
||||
def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, example_mobi, example_pdf,
|
||||
singlefile_contents_factory):
|
||||
singlefile_contents_factory, make_files_structure):
|
||||
"""Test that the most important file is returned from a list of files."""
|
||||
srt_file3 = srt_file3.rename(test_directory / f'{video_file.stem}.srt')
|
||||
# Same primary_file no matter the order.
|
||||
|
|
@ -612,6 +632,11 @@ def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, e
|
|||
assert lib.get_primary_file([example_pdf, example_mobi]) == example_pdf
|
||||
assert lib.get_primary_file([example_pdf, example_mobi, example_epub]) == example_epub
|
||||
|
||||
# No primary file in this group.
|
||||
foo, bar = make_files_structure({'foo': 'text', 'bar': None})
|
||||
with pytest.raises(NoPrimaryFile):
|
||||
assert lib.get_primary_file([foo, bar])
|
||||
|
||||
|
||||
def test_get_refresh_progress(test_client, test_session):
|
||||
request, response = test_client.get('/api/files/refresh_progress')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue