From 9dfaabdb9b9820945d16b521e7fb877a54986354 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 29 Dec 2022 09:36:24 -0700 Subject: [PATCH] Not sending refresh events for channel downloads. --- modules/videos/channel/test/test_api.py | 6 +++++- modules/videos/downloader.py | 2 +- modules/videos/models.py | 4 ++-- wrolpi/conftest.py | 3 ++- wrolpi/files/lib.py | 9 +++++---- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index 72353bd9..b180662e 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -255,7 +255,8 @@ def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_direc @pytest.mark.asyncio -async def test_download_channel_no_refresh(test_session, download_channel, video_download_manager, video_factory): +async def test_channel_download_requires_refresh(test_session, download_channel, video_download_manager, video_factory, + events_history): """A Channel cannot be downloaded until it has been refreshed. Videos already downloaded are not downloaded again.""" @@ -301,6 +302,9 @@ async def test_download_channel_no_refresh(test_session, download_channel, video # Only the missing video was downloaded. assert downloaded_urls == ['https://example.com/2'] + # Should not send refresh events because downloads are automated. + assert events_history == [] + def test_channel_post_directory(test_session, test_client, test_directory): """A Channel can be created with or without an existing directory.""" diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 02cf0753..f71783d4 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -137,7 +137,7 @@ class ChannelDownloader(Downloader, ABC): channel = get_channel(session, channel_id=channel_id, return_dict=False) if not channel.videos or not channel.refreshed: logger.warning(f'Refreshing videos in {channel.directory} because {channel} has no video records!') - await channel.refresh_files() + await channel.refresh_files(send_events=False) session.commit() # Commit the refresh. session.refresh(channel) diff --git a/modules/videos/models.py b/modules/videos/models.py index 036116d4..a61d9581 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -479,14 +479,14 @@ class Channel(ModelHelper, Base): ) return statistics - async def refresh_files(self): + async def refresh_files(self, send_events: bool = True): """Refresh all files within this Channel's directory. Mark this channel as refreshed.""" logger.debug('Channel.refresh_files refresh_files') # Get this Channel's ID for later. Refresh may take a long time. self_id = self.id # Refresh all files within this channel's directory first. - await refresh_directory_files_recursively(self.directory) + await refresh_directory_files_recursively(self.directory, send_events=send_events) # Apply any info_json (discover censored videos, etc.) second. from modules.videos.common import apply_info_json diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 888fff1f..b54b4931 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -321,7 +321,8 @@ def mock_create_subprocess_shell(): def events_history(): """Give each test it's own Events history.""" with mock.patch('wrolpi.events.EVENTS_HISTORY', list()): - yield + from wrolpi.events import EVENTS_HISTORY + yield EVENTS_HISTORY FLAGS_LOCK = multiprocessing.Lock() diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 9aff5359..78fcef40 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -22,7 +22,6 @@ from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidFile from wrolpi.events import Events from wrolpi.files.models import File -from wrolpi import flags from wrolpi.vars import PYTEST, FILE_REFRESH_CHUNK_SIZE try: @@ -359,7 +358,7 @@ async def refresh_files(): @limit_concurrent(1) @wrol_mode_check @cancelable_wrapper -async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str]): +async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str], send_events: bool = True): """Upsert and index all files within a directory (recursively). Any records of the files that are no longer in the directory will be removed.""" @@ -369,7 +368,8 @@ async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str raise ValueError(f'Cannot refresh files of a file: {directory=}') relative_path = str(get_relative_to_media_directory(directory)) - Events.send_directory_refresh_started(f'Refresh of {repr(relative_path)} has started.') + if send_events: + Events.send_directory_refresh_started(f'Refresh of {repr(relative_path)} has started.') # All Files older than this will be removed. idempotency = now() @@ -395,7 +395,8 @@ async def refresh_directory_files_recursively(directory: Union[pathlib.Path, str apply_after_refresh() await apply_indexers() - Events.send_directory_refresh_completed(f'Refresh of {repr(relative_path)} has completed.') + if send_events: + Events.send_directory_refresh_completed(f'Refresh of {repr(relative_path)} has completed.') refresh_logger.info(f'Done refreshing files in {directory}')