From 9ca099267bdb20cf00e4f57787646ba81c0ec352 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 5 Apr 2023 10:00:08 -0600 Subject: [PATCH] Refreshing file when attempting to add Tag to file that has not been refreshed. Adding url to skip list when deleting download. --- modules/archive/lib.py | 2 +- wrolpi/downloader.py | 7 ++----- wrolpi/files/api.py | 8 ++++---- wrolpi/files/lib.py | 31 +++++++++++++++++++------------ 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index bd8bb3d9..5e1fb3e9 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -113,7 +113,7 @@ async def request_archive(url: str) -> Tuple[str, Optional[str], Optional[str]]: # Report the error from the archive service. raise Exception(f'Received error from archive service: {error}') - readability = contents['readability'] + readability = contents.get('readability') # Compressed base64 singlefile = contents['singlefile'] screenshot = contents['screenshot'] diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index bda29712..f010c785 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -757,13 +757,10 @@ class DownloadManager: """Delete a Download. Returns True if a Download was deleted, otherwise return False.""" download = self.get_download(session, id_=download_id) if download: + url = download.url session.delete(download) session.commit() - # Save the config because a download was deleted. - if PYTEST: - save_downloads_config() - else: - background_task(save_downloads_config()) + self.add_to_skip_list(url) return True return False diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 17909899..b600aef2 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -111,7 +111,7 @@ def post_directories(_, body: schema.DirectoriesRequest): @bp.post('/tag') @validate(schema.TagFileGroupPost) -def post_tag_file_group(_, body: schema.TagFileGroupPost): +async def post_tag_file_group(_, body: schema.TagFileGroupPost): if not body.tag_id and not body.tag_name: return json_response( dict(error='tag_id and tag_name cannot both be empty'), @@ -123,12 +123,12 @@ def post_tag_file_group(_, body: schema.TagFileGroupPost): HTTPStatus.BAD_REQUEST, ) - lib.add_file_group_tag(body.file_group_id, body.file_group_primary_path, body.tag_name, body.tag_id) + await lib.add_file_group_tag(body.file_group_id, body.file_group_primary_path, body.tag_name, body.tag_id) return response.empty(HTTPStatus.CREATED) @bp.post('/untag') @validate(schema.TagFileGroupPost) -def post_untag_file_group(_, body: schema.TagFileGroupPost): - lib.remove_file_group_tag(body.file_group_id, body.file_group_primary_path, body.tag_name, body.tag_id) +async def post_untag_file_group(_, body: schema.TagFileGroupPost): + await lib.remove_file_group_tag(body.file_group_id, body.file_group_primary_path, body.tag_name, body.tag_id) return response.empty(HTTPStatus.NO_CONTENT) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 42aa3d43..5dc4d6d6 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -19,7 +19,7 @@ from wrolpi import flags from wrolpi.cmd import which from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \ partition, cancelable_wrapper, \ - get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup + get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task from wrolpi.dates import now, from_timestamp from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag @@ -764,17 +764,24 @@ def group_files_by_stem(files: List[pathlib.Path], pre_sorted: bool = False) -> yield group -def _get_tag_and_file_group(file_group_id: int, file_group_primary_path: str, tag_name: str, tag_id: int, - session: Session): +async def _get_tag_and_file_group(file_group_id: int, file_group_primary_path: str, tag_name: str, tag_id: int, + session: Session): if file_group_id: file_group: FileGroup = session.query(FileGroup).filter_by(id=file_group_id).one_or_none() if not file_group: raise UnknownFile(f'Cannot find FileGroup with id {file_group_id}') elif file_group_primary_path: - path = str(get_media_directory() / file_group_primary_path) - file_group: FileGroup = session.query(FileGroup).filter_by(primary_path=path).one_or_none() + path = get_media_directory() / file_group_primary_path + file_group: FileGroup = session.query(FileGroup).filter_by(primary_path=str(path)).one_or_none() + if not file_group and path.is_file(): + # File may not have been refreshed. + await refresh_discover_paths([path]) + session.flush() + file_group: FileGroup = session.query(FileGroup).filter_by(primary_path=str(path)).one_or_none() + background_task(refresh_files([path])) + if not file_group: - raise UnknownFile(f'Cannot find FileGroup with primary_path {file_group_primary_path}') + raise UnknownFile(f'Cannot find FileGroup with primary_path {repr(str(file_group_primary_path))}') else: raise UnknownFile(f'Cannot find FileGroup without id or primary_path') @@ -793,17 +800,17 @@ def _get_tag_and_file_group(file_group_id: int, file_group_primary_path: str, ta @optional_session(commit=True) -def add_file_group_tag(file_group_id: int, file_group_primary_path: str, tag_name: str, tag_id: int, - session: Session = None) -> TagFile: - file_group, tag = _get_tag_and_file_group(file_group_id, file_group_primary_path, tag_name, tag_id, session) +async def add_file_group_tag(file_group_id: int, file_group_primary_path: str, tag_name: str, tag_id: int, + session: Session = None) -> TagFile: + file_group, tag = await _get_tag_and_file_group(file_group_id, file_group_primary_path, tag_name, tag_id, session) tag_file = file_group.add_tag(tag, session) return tag_file @optional_session(commit=True) -def remove_file_group_tag(file_group_id: int, file_group_primary_path: str, tag_name: str, tag_id: int, - session: Session = None): - file_group, tag = _get_tag_and_file_group(file_group_id, file_group_primary_path, tag_name, tag_id, session) +async def remove_file_group_tag(file_group_id: int, file_group_primary_path: str, tag_name: str, tag_id: int, + session: Session = None): + file_group, tag = await _get_tag_and_file_group(file_group_id, file_group_primary_path, tag_name, tag_id, session) file_group.remove_tag(tag, session)