From e7582944b959cdb8723272390ac74fa7f23fcb98 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 12 Sep 2023 20:24:02 -0600 Subject: [PATCH] Channels not in the channels config are deleted on import. --- modules/videos/lib.py | 55 ++++++++---------------------- modules/videos/models.py | 2 +- modules/videos/test/test_common.py | 39 +++++++++++++++++++++ modules/videos/test/test_lib.py | 27 --------------- 4 files changed, 54 insertions(+), 69 deletions(-) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 49387be7..9c4e3bba 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -12,16 +12,14 @@ from yt_dlp import YoutubeDL from wrolpi import before_startup, dates from wrolpi.captions import extract_captions -from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_refresh_cleanup, limit_concurrent +from wrolpi.common import ConfigFile, get_media_directory, register_refresh_cleanup, limit_concurrent from wrolpi.dates import Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.errors import UnknownDirectory -from wrolpi.files.models import FileGroup -from wrolpi.tags import Tag from wrolpi.vars import PYTEST from .common import is_valid_poster, convert_image, \ generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \ - get_no_channel_directory, get_video_duration + get_video_duration from .models import Channel, Video logger = logger.getChild(__name__) @@ -387,7 +385,8 @@ def import_channels_config(): channel_import_logger.info('Importing videos config') try: config = get_channels_config() - channels, favorites = config.channels, config.favorites + channels = config.channels + channel_directories = [i['directory'] for i in channels] save_config = False with get_db_session(commit=True) as session: @@ -412,6 +411,9 @@ def import_channels_config(): # Channel not yet in the DB, add it. channel = Channel(directory=directory) session.add(channel) + session.flush([channel, ]) + # TODO refresh the files in the channel. + logger.warning(f'Creating new Channel from config {channel}') # Only name and directory are required channel.name = data['name'] @@ -443,46 +445,17 @@ def import_channels_config(): f' download_frequency={channel.download_frequency}' ) + with get_db_session(commit=True) as session: + # Delete any Channels that were deleted from the config. + for channel in session.query(Channel): + if str(channel.directory) not in channel_directories: + logger.warning(f'Deleting {channel} because it is not in the config.') + channel.delete_with_videos() + if save_config: # Information about the channel was fetched, store it. save_channels_config() - with get_db_session(commit=True) as session: - channels_by_link = {sanitize_link(i.name): i for i in session.query(Channel)} - for directory_, favorites in favorites.items(): - if directory_ != 'NO CHANNEL': - # A Channel's directory is saved (in the config) relative to the media directory. - directory = get_media_directory() / directory_ - channel: Channel = session.query(Channel).filter_by(directory=directory).one_or_none() - if not channel: - # Directory may be the outdated "link". - # TODO remove these old favorites after beta. - channel = channels_by_link.get(directory_) - if not channel: - channel_import_logger.warning(f'Cannot find channel {directory=} for favorites!') - continue - channel_dir = channel.directory - else: - channel_dir = get_no_channel_directory() - - favorites_tag = Tag.find_by_name('Favorite') - if not favorites_tag: - channel_import_logger.warning(f'Cannot tag favorite videos without Favorite tag!') - return - - # Set favorite Videos of this Channel. - for video_path, data in favorites.items(): - # Favorite in the config is the name of the video_path. Add the channel directory onto this - # video_path, so we can match the complete path for the Video. - video_path = channel_dir / video_path - file_group: FileGroup = FileGroup.get_by_path(video_path, session) - if file_group and file_group.tag_files: - # Video is already tagged or favorited. - pass - elif file_group: - file_group.add_tag(favorites_tag, session) - else: - channel_import_logger.warning(f'Cannot find video to favorite: {video_path}') except Exception as e: channel_import_logger.warning('Failed to load channels config!', exc_info=e) if PYTEST: diff --git a/modules/videos/models.py b/modules/videos/models.py index c75b1f18..8a822891 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -400,7 +400,7 @@ class Channel(ModelHelper, Base): videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id') def __repr__(self): - return f'' + return f'' def __eq__(self, other): if isinstance(other, Channel): diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index f59266ab..d3c2321a 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -185,6 +185,45 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c assert next_download == str(download.next_download) +def test_import_channel_delete_missing_channels(test_session, channel_factory, test_channels_config): + """The Channel import function deletes any Channels that are not in the config.""" + # Create a DB and config with two Channels. + channel1 = channel_factory(source_id='foo') + channel2 = channel_factory(source_id='bar') + test_session.commit() + # Write Channels to the config file. + save_channels_config() + test_session.delete(channel1) + test_session.delete(channel2) + test_session.commit() + + # Importing the config creates two Channels. + import_channels_config() + assert len(test_session.query(Channel).all()) == 2 + assert str(channel1.directory) in test_channels_config.read_text() + assert str(channel2.directory) in test_channels_config.read_text() + + # Delete channel2 from the config file. + config = get_channels_config() + config_dict = config.dict() + config_dict['channels'] = [i for i in config.channels if i['directory'] != str(channel2.directory)] + config.update(config_dict) + assert str(channel1.directory) in test_channels_config.read_text() + assert str(channel2.directory) not in test_channels_config.read_text() + + # Importing the config deletes the Channel record. + import_channels_config() + assert len(test_session.query(Channel).all()) == 1 + assert str(channel1.directory) in test_channels_config.read_text() + assert str(channel2.directory) not in test_channels_config.read_text() + + # Saving and importing does not change anything. + save_channels_config() + import_channels_config() + assert str(channel1.directory) in test_channels_config.read_text() + assert str(channel2.directory) not in test_channels_config.read_text() + + def test_check_for_video_corruption(video_file, test_directory): # The test video is not corrupt. assert common.check_for_video_corruption(video_file) is False diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 23320822..5248f08a 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -149,30 +149,3 @@ async def test_orphaned_files(test_session, make_files_structure, test_directory vid2_poster_path, vid3_poster_path, ]) - - -@pytest.mark.asyncio -async def test_import_channels_config_favorites_tags(test_session, test_directory, test_channels_config, video_factory, - simple_channel): - """Video favorites have been moved to Tags. Any Favorited video should be tagged with Favorite tag.""" - test_session.add(Tag(name='Favorite')) - video1 = video_factory(channel_id=simple_channel.id) - video2 = video_factory(channel_id=simple_channel.id) - test_session.commit() - - # Favorite video1 in config. - channels_config = lib.get_channels_config() - channels_config.favorites = { - str(simple_channel.directory.relative_to(test_directory)): { - str(video1.video_path.relative_to(simple_channel.directory)): now().isoformat(), - } - } - - # The import function will tag the Video with Favorite. - lib.import_channels_config() - - # video1 has Favorite tag. It is also in the config. - assert video1.file_group.tag_files and video1.file_group.tag_files[0].tag.name == 'Favorite' - assert str(video1.file_group.primary_path.relative_to(test_directory)) in test_channels_config.read_text() - # video2 was not tagged. - assert not video2.file_group.tag_files