diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index cb215b7d..07b20446 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -13,7 +13,7 @@ import zipfile from abc import ABC from datetime import datetime from itertools import zip_longest -from typing import List, Callable, Dict, Sequence +from typing import List, Callable, Dict, Sequence, Union from typing import Tuple, Set from unittest import mock from unittest.mock import MagicMock @@ -375,7 +375,12 @@ def example_mobi(test_directory) -> pathlib.Path: @pytest.fixture def make_files_structure(test_directory) -> Callable: - def create_files(paths: List, file_groups: bool = False, session: Session = None) -> List[pathlib.Path]: + """ + A fixture which creates test files passed to it. If a list is provided, empty files will be created at those + locations. If a dict is provided, files will be created containing the value of the dict item. + """ + def create_files(paths: Union[List, Dict], file_groups: bool = False, session: Session = None) -> List[ + pathlib.Path]: files = [] @iterify(list) @@ -404,7 +409,7 @@ def make_files_structure(test_directory) -> Callable: for idx, path in enumerate(files): fg = FileGroup.from_paths(session, path) files[idx] = fg - return files + return sorted(files) if isinstance(paths, dict) else files return create_files diff --git a/wrolpi/errors.py b/wrolpi/errors.py index 83d22a7e..496ca8a0 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -137,3 +137,9 @@ class InvalidConfig(APIError): code = 'INVALID_CONFIG' summary = 'Config is invalid' status = HTTPStatus.BAD_REQUEST + + +class NoPrimaryFile(APIError): + code = 'NO_PRIMARY_FILE' + summary = 'Could not find a primary file in the provided group' + status = HTTPStatus.INTERNAL_SERVER_ERROR diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index f324642a..04381cdd 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -25,7 +25,8 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk from wrolpi.dates import now, from_timestamp from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session -from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged +from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \ + NoPrimaryFile from wrolpi.events import Events from wrolpi.files.models import FileGroup, Directory from wrolpi.lang import ISO_639_CODES @@ -165,7 +166,9 @@ async def delete(*paths: Union[str, pathlib.Path]): # Search for any FileGroups that have been tagged under this directory. query = query.filter(FileGroup.primary_path.like(f'{path}/%')) for (file_group, tag_file) in query: - raise FileGroupIsTagged(f"Cannot delete {file_group} because it is tagged") + if any(i for i in paths if i in file_group.my_paths()): + # File that will be deleted is in a Tagged FileGroup. + raise FileGroupIsTagged(f"Cannot delete {file_group} because it is tagged") for path in paths: if path.is_dir(): delete_directory(path, recursive=True) @@ -179,11 +182,6 @@ async def delete(*paths: Union[str, pathlib.Path]): background_task(coro) -FILE_NAME_REGEX = re.compile(r'[_ .]') - -FILE_BIN = which('file', '/usr/bin/file') - - def _mimetype_suffix_map(path: Path, mimetype: str): """Special handling for mimetypes. Python Magic may not return the correct mimetype.""" from wrolpi.files.ebooks import MOBI_MIMETYPE @@ -228,6 +226,9 @@ def _mimetype_suffix_map(path: Path, mimetype: str): return mimetype +FILE_BIN = which('file', '/usr/bin/file') + + @functools.lru_cache(maxsize=10_000) def get_mimetype(path: Path) -> str: """Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command.""" @@ -299,8 +300,10 @@ def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]: return files -def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> Union[pathlib.Path, List[pathlib.Path]]: - """Given a list of files, return the file that we can model or index.""" +def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> pathlib.Path: + """Given a list of files, return the file that we can model or index. + + @raise NoPrimaryFile: If not primary file can be found.""" from modules.archive.lib import is_singlefile_file if len(files) == 0: @@ -332,10 +335,7 @@ def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> U if mimetype.startswith('image/'): return file - refresh_logger.debug(f'Cannot find primary file for group: {files}') - - # Can't find a typical primary file. - return files + raise NoPrimaryFile(f'Cannot find primary file for group: {files}') def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): @@ -355,10 +355,10 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): values = dict() for group in grouped: # The primary file is the video/SingleFile/epub, etc. - primary_path_or_paths = get_primary_file(group) - if isinstance(primary_path_or_paths, pathlib.Path): + try: + primary_path = get_primary_file(group) # Multiple files in this group. - primary_path: pathlib.Path = primary_path_or_paths + primary_path: pathlib.Path = primary_path # The primary mimetype allows modelers to find its file_groups. mimetype = get_mimetype(primary_path) # The group uses a common modification_datetime so the group will be re-indexed when any of it's files @@ -368,9 +368,9 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): files = json.dumps(_paths_to_files_dict(group)) values[primary_path] = (modification_datetime, mimetype, size, files) non_primary_files = {i for i in group if i != primary_path} - else: + except NoPrimaryFile: # Cannot find primary path, create a `file_group` for each file. - for primary_path in primary_path_or_paths: + for primary_path in group: primary_path: pathlib.Path # The primary mimetype allows modelers to find its file_groups. mimetype = get_mimetype(primary_path) diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index 70eb3419..719f0d9f 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -513,7 +513,7 @@ async def test_directory_crud(test_session, test_async_client, test_directory, a @pytest.mark.asyncio async def test_move(test_session, test_directory, make_files_structure, test_async_client): - foo, bar = make_files_structure({ + baz, bar = make_files_structure({ 'foo/bar.txt': 'bar', 'baz.txt': 'baz', }) @@ -529,14 +529,14 @@ async def test_move(test_session, test_directory, make_files_structure, test_asy content = dict(paths=['foo', 'baz.txt'], destination='qux') request, response = await test_async_client.post('/api/files/move', content=json.dumps(content)) assert response.status_code == HTTPStatus.NO_CONTENT - assert not foo.exists() + assert not bar.exists() assert (test_directory / 'qux/foo/bar.txt').is_file() # mv foo/bar.txt qux content = dict(paths=['qux/foo/bar.txt', ], destination='qux') request, response = await test_async_client.post('/api/files/move', content=json.dumps(content)) assert response.status_code == HTTPStatus.NO_CONTENT - assert not foo.exists() + assert not bar.exists() assert (test_directory / 'qux/bar.txt').is_file() assert (test_directory / 'qux/foo').is_dir() diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 6276c1d1..7ea9d4e8 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -14,7 +14,7 @@ from PIL import Image from wrolpi.common import timer from wrolpi.dates import now, from_timestamp -from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged +from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile from wrolpi.files import lib, indexers from wrolpi.files.models import FileGroup from wrolpi.tags import TagFile @@ -22,7 +22,7 @@ from wrolpi.vars import PROJECT_DIR @pytest.mark.asyncio -async def test_delete(test_session, make_files_structure, test_directory): +async def test_delete_file(test_session, make_files_structure, test_directory): """ File in the media directory can be deleted. """ @@ -54,7 +54,7 @@ async def test_delete(test_session, make_files_structure, test_directory): @pytest.mark.asyncio -async def test_delete_multiple(test_session, make_files_structure, test_directory): +async def test_delete_file_multiple(test_session, make_files_structure, test_directory): """Multiple files can be deleted at once.""" foo, bar, baz = make_files_structure([ 'archives/foo.txt', @@ -72,7 +72,27 @@ async def test_delete_multiple(test_session, make_files_structure, test_director @pytest.mark.asyncio -async def test_delete_link(test_session, test_directory): +async def test_delete_file_names(test_session, make_files_structure, test_directory, tag_factory): + """Will not refuse to delete a file that shares the name of a nearby file when they are in different FileGroups.""" + foo, foo1 = make_files_structure({ + 'archives/foo': 'text', + 'archives/foo1': 'text', + }) + foo_fg = FileGroup.from_paths(test_session, foo) + foo1_fg = FileGroup.from_paths(test_session, foo1) + tag = tag_factory() + foo1_fg.add_tag(tag) + test_session.commit() + assert foo.is_file() + assert foo1.is_file() + + await lib.delete('archives/foo') + assert not foo.is_file() + assert foo1.is_file() + + +@pytest.mark.asyncio +async def test_delete_file_link(test_session, test_directory): """Links can be deleted.""" foo, bar = test_directory / 'foo', test_directory / 'bar' foo.touch() @@ -187,7 +207,7 @@ async def test_refresh_empty_media_directory(test_session, test_directory): @pytest.mark.asyncio async def test__upsert_files(test_session, make_files_structure, test_directory, assert_file_groups, video_file, srt_file3): - bar, baz = make_files_structure({ + baz, bar = make_files_structure({ 'dir1/bar.txt': None, 'baz.txt': 'baz file', }) @@ -592,7 +612,7 @@ def test_group_files_by_stem(make_files_structure, test_directory): def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, example_mobi, example_pdf, - singlefile_contents_factory): + singlefile_contents_factory, make_files_structure): """Test that the most important file is returned from a list of files.""" srt_file3 = srt_file3.rename(test_directory / f'{video_file.stem}.srt') # Same primary_file no matter the order. @@ -612,6 +632,11 @@ def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, e assert lib.get_primary_file([example_pdf, example_mobi]) == example_pdf assert lib.get_primary_file([example_pdf, example_mobi, example_epub]) == example_epub + # No primary file in this group. + foo, bar = make_files_structure({'foo': 'text', 'bar': None}) + with pytest.raises(NoPrimaryFile): + assert lib.get_primary_file([foo, bar]) + def test_get_refresh_progress(test_client, test_session): request, response = test_client.get('/api/files/refresh_progress')