From 2f0d2f9b95581abdc6fdc881b0b9da5c88d2971d Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 13 Dec 2022 11:51:08 -0700 Subject: [PATCH] Testing that ebook index is only performed once. --- wrolpi/files/api.py | 2 +- wrolpi/files/ebooks.py | 9 ++++----- wrolpi/files/lib.py | 10 ++++++++-- wrolpi/files/test/test_ebooks.py | 6 ++++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 889d520e..0398856d 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -95,7 +95,7 @@ async def refresh_directory(_: Request, body: schema.DirectoryRefreshRequest): @bp.post('/refresh/list') @openapi.definition( - summary='Index all files in the provided list', + summary='Refresh and index all files in the provided list', body=schema.FilesRefreshListRequest, ) @validate(schema.FilesRefreshListRequest) diff --git a/wrolpi/files/ebooks.py b/wrolpi/files/ebooks.py index 43af4be3..a21fb090 100644 --- a/wrolpi/files/ebooks.py +++ b/wrolpi/files/ebooks.py @@ -198,9 +198,7 @@ class EBook(ModelHelper, Base): def model_ebook(session: Session, ebook_file: File, files: List[File]) -> EBook: - """ - Creates an EBook model based off a File. Searches for it's cover in the provided `files`. - """ + """Creates an EBook model based off a File. Searches for it's cover in the provided `files`.""" # Multiple formats may share this cover. cover_file = next((i for i in files if i.mimetype.split('/')[0] == 'image'), None) @@ -213,8 +211,9 @@ def model_ebook(session: Session, ebook_file: File, files: List[File]) -> EBook: ebook.ebook_file.do_index() - index = size != ebook.size or cover_file != ebook.cover_file or not ebook.title - if not ebook.ebook_file.indexed or index: + # Only index if it hasn't been done, or if the file has changed. + changed = size != ebook.size or cover_file != ebook.cover_file or not ebook.title + if not ebook.ebook_file.indexed or changed: try: # Only read the contents of the file if it has changed. data = extract_ebook_data(ebook_file.path, ebook_file.mimetype) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 3a520e2a..7cbf0b22 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -102,6 +102,8 @@ def _mimetype_suffix_map(path: Path, mimetype: str): return MOBI_MIMETYPE if mimetype == 'text/plain' and suffix.endswith('.json'): return 'application/json' + if mimetype == 'text/plain' and suffix.endswith('.vtt'): + return 'text/vtt' return mimetype @@ -177,7 +179,7 @@ async def _refresh_files_list(paths: List[pathlib.Path], idempotency: datetime.d if idx > 0: parent = all_files[0].path.parent - refresh_logger.debug(f'Committed chunk of {len(chunk)} files in {parent}') + refresh_logger.debug(f'Committed chunk of {len(chunk)} files. Refreshing near {parent}') # Sleep between chunks to catch cancel. await asyncio.sleep(0) @@ -234,7 +236,7 @@ async def refresh_files_list(paths: List[str], include_files_near: bool = True): media directory. include_files_near: If true, will also refresh files that share the stem of the provided files. """ - refresh_logger.info(f'Refreshing {len(paths)} files') + refresh_logger.info(f'Refreshing {len(paths)} files list') media_directory = get_media_directory() @@ -273,6 +275,10 @@ async def refresh_files_list(paths: List[str], include_files_near: bool = True): deleted_paths = list(map(str, deleted_paths)) curs.execute('DELETE FROM file WHERE path = ANY(%(deleted_paths)s)', dict(deleted_paths=deleted_paths)) + apply_after_refresh() + + refresh_logger.info('Done refreshing files list') + @limit_concurrent(1) # Only one refresh at a time. @wrol_mode_check diff --git a/wrolpi/files/test/test_ebooks.py b/wrolpi/files/test/test_ebooks.py index 5180b566..9271b980 100644 --- a/wrolpi/files/test/test_ebooks.py +++ b/wrolpi/files/test/test_ebooks.py @@ -1,6 +1,7 @@ import json import shutil from http import HTTPStatus +from unittest import mock import pytest @@ -44,6 +45,11 @@ async def test_index(test_session, test_directory, example_epub, example_mobi): await refresh_files() assert test_session.query(EBook).count() == 0 + # Extract/index should only be done once. + with mock.patch('wrolpi.files.ebooks.extract_ebook_data') as mock_extract_ebook_data: + mock_extract_ebook_data.side_effect = Exception('extract_ebook_data should not be called again') + await refresh_files() + def test_search(test_session, test_client, example_epub): """Ebooks are handled in File search results."""