From bedcc3a5e9b1a6c45d380cc19a2bfeef1b8c9785 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Wed, 5 Jan 2022 12:52:23 -0700 Subject: [PATCH] Bugfix/favorites import (#71) * Favorite videos are saved even if the DB is wiped. * Logging when importing. * Warning when favorite video cannot be found. * Handling when a favorited video has no channel. * Fixing link name in log. * vid2 was not in a channel. Signed-off-by: Roland --- modules/videos/common.py | 47 +++++++++++++++++---- modules/videos/conftest.py | 5 ++- modules/videos/downloader.py | 7 +--- modules/videos/lib.py | 21 ++++++++-- modules/videos/models.py | 19 +++++---- modules/videos/test/test_common.py | 67 +++++++++++++++++++++++++++++- wrolpi/common.py | 4 +- wrolpi/conftest.py | 5 ++- 8 files changed, 143 insertions(+), 32 deletions(-) diff --git a/modules/videos/common.py b/modules/videos/common.py index fb7f03a0..910b2f41 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -11,6 +11,7 @@ from typing import Union, Tuple, List, Set, Iterable, Optional import PIL from PIL import Image from sqlalchemy.orm import Session +from yt_dlp import YoutubeDL from wrolpi import before_startup from wrolpi.common import logger, get_config, iterify, chunks, get_media_directory, \ @@ -58,8 +59,10 @@ class ConfigError(Exception): @before_startup def import_videos_config(): """Import channel settings to the DB. Existing channels will be updated.""" + logger.info('Importing videos config') try: - config = load_channels_config() + config = get_config() + config, favorites = config['channels'], config.get('favorites', {}) with get_db_session(commit=True) as session: for link in config: @@ -87,19 +90,35 @@ def import_videos_config(): if not channel.source_id and channel.url: # If we can download from a channel, we must have its source_id. - from .downloader import get_channel_source_id channel.source_id = get_channel_source_id(channel.url) session.add(channel) + with get_db_session(commit=True) as session: + for link, favorites in favorites.items(): + if link != 'NO CHANNEL': + channel: Channel = session.query(Channel).filter_by(link=link).one_or_none() + if not channel: + logger.warning(f'Cannot find channel {link=} for favorites!') + continue + channel_dir = channel.directory.path + else: + from .downloader import get_no_channel_directory + channel_dir = get_no_channel_directory() + # Set favorite Videos of this Channel. - favorites = config[link].get('favorites', {}) - if favorites: - videos = session.query(Video).filter(Video.source_id.in_(favorites.keys())) - for video in videos: - video.favorite = favorites[video.source_id]['favorite'] - except Exception: - logger.warning('Failed to load channels config!', exc_info=True) + for video_path, data in favorites.items(): + # Favorite in the config is the name of the video_path. Add the channel directory onto this + # video_path, so we can match the complete path for the Video. + video_path = channel_dir / video_path + video = session.query(Video).filter_by(video_path=video_path).one_or_none() + # If no Video is found, it may be that we need to refresh. + if video: + video.favorite = data['favorite'] + else: + logger.warning(f'Cannot find video to favorite: {video_path}') + except Exception as e: + logger.warning('Failed to load channels config!', exc_info=e) VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'} @@ -545,3 +564,13 @@ def minimize_video(video: dict) -> dict: video['info_json'] = minimize_video_info_json(video['info_json']) return video + + +YDL = YoutubeDL() +YDL.params['logger'] = logger.getChild('youtube-dl') +YDL.add_default_info_extractors() + + +def get_channel_source_id(url: str) -> str: + channel_info = YDL.extract_info(url, download=False, process=False) + return channel_info.get('uploader_id') or channel_info['id'] diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 28d5d738..b3026c33 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -101,7 +101,10 @@ def video_file(test_directory) -> pathlib.Path: def video_factory(test_session, test_directory): def _(channel_id: int = None): title = str(uuid4()) - path = str(test_directory / f'{title}.mp4') + if channel_id: + path = str(test_directory / f'{title}.mp4') + else: + path = str(test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4') video = Video(video_path=path, title=title, channel_id=channel_id, source_id=title) test_session.add(video) return video diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 949e0896..65c4f6c3 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -19,7 +19,7 @@ from wrolpi.downloader import Downloader, Download from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel -from .common import load_downloader_config, apply_info_json +from .common import load_downloader_config, apply_info_json, get_channel_source_id from .lib import upsert_video, _refresh_videos from .models import Video, Channel from .video_url_resolver import video_url_resolver @@ -469,8 +469,3 @@ def _skip_download(error): if msg in error_str: return True return False - - -def get_channel_source_id(url: str) -> str: - channel_info = YDL.extract_info(url, download=False, process=False) - return channel_info.get('uploader_id') or channel_info['id'] diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 4ab3fc80..ef81e4e4 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -3,12 +3,13 @@ import html import json import pathlib import re +from collections import defaultdict from typing import Tuple from uuid import uuid1 from sqlalchemy.orm import Session -from wrolpi.common import logger, chunks +from wrolpi.common import logger, chunks, get_config from wrolpi.common import save_settings_config from wrolpi.db import get_db_curs, get_db_session from wrolpi.vars import PYTEST @@ -264,10 +265,22 @@ def get_channels_config(session: Session) -> dict: """ channels = session.query(Channel).order_by(Channel.link).all() channels = {i.link: i.config_view() for i in channels} - return dict(channels=channels) + + # Get all Videos that are favorites. Store them in their own config section, so they can be preserved if a channel + # is deleted or the DB is wiped. + favorite_videos = session.query(Video).filter(Video.favorite != None, Video.video_path != None).all() # noqa + favorites = defaultdict(lambda: {}) + for video in favorite_videos: + if video.channel: + favorites[video.channel.link][video.video_path.path.name] = dict(favorite=video.favorite) + else: + favorites['NO CHANNEL'][video.video_path.path.name] = dict(favorite=video.favorite) + favorites = dict(favorites) + + return dict(channels=channels, favorites=favorites) -def save_channels_config(session=None): +def save_channels_config(session=None, preserve_favorites: bool = True): """ Pull the Channel information from the DB, save it to the config. """ @@ -276,6 +289,8 @@ def save_channels_config(session=None): else: with get_db_session() as session: config = get_channels_config(session) + if preserve_favorites: + config['favorites'].update(get_config().get('favorites', {})) save_settings_config(config) diff --git a/modules/videos/models.py b/modules/videos/models.py index 1797ec9a..43fa213c 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -1,7 +1,7 @@ import json from datetime import datetime from pathlib import Path -from typing import Optional, List +from typing import Optional from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, Computed from sqlalchemy.orm import relationship, Session @@ -95,7 +95,17 @@ class Video(ModelHelper, Base): if not video_files: raise UnknownFile('No video files were deleted') + needs_save = bool(self.favorite) + self.add_to_skip_list() + self.favorite = None + session = Session.object_session(self) + session.commit() + + if needs_save: + # Save the Channels config. This video was a favorite and that needs to be removed. + from modules.videos.lib import save_channels_config + save_channels_config(preserve_favorites=False) def add_to_skip_list(self): """ @@ -300,13 +310,6 @@ class Channel(ModelHelper, Base): source_id=self.source_id, url=self.url or '', ) - - session = Session.object_session(self) - favorites = session.query(Video).filter(Video.favorite != None, Video.channel_id == self.id).all() # noqa - favorites: List[Video] - if favorites: - config['favorites'] = {i.video_path.path.name: {'favorite': i.favorite} for i in favorites} - return config def get_relative_path(self, path: Path, exists: bool = True): diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 2a143ec6..16d4179e 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -2,6 +2,8 @@ import json import pathlib import subprocess import tempfile +from datetime import datetime +from typing import List from unittest import mock from unittest.mock import Mock, MagicMock @@ -10,13 +12,15 @@ from PIL import Image from modules.videos.models import Channel, Video from modules.videos.test.common import create_channel_structure -from wrolpi.common import get_absolute_media_path +from wrolpi.common import get_absolute_media_path, get_config +from wrolpi.dates import local_timezone from wrolpi.db import get_db_session from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI from wrolpi.vars import PROJECT_DIR from ..common import get_matching_directories, convert_image, bulk_validate_posters, remove_duplicate_video_paths, \ apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \ - get_bulk_video_info_json + get_bulk_video_info_json, import_videos_config +from ..lib import save_channels_config class TestCommon(TestAPI): @@ -405,3 +409,62 @@ async def test_missing_title(test_session, simple_video): test_session.commit() await get_bulk_video_info_json([simple_video.id]) assert simple_video.title == 'simple_video' + + +def test_import_favorites(test_session, simple_channel, video_factory): + """ + A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be + favorited again. + + A Video's favorited status can only be cleared by calling `Video.delete`. + """ + video_factory(channel_id=simple_channel.id) # never favorited + vid2 = video_factory(channel_id=None) # has no channel + vid3 = video_factory(channel_id=simple_channel.id) + favorite = vid2.favorite = local_timezone(datetime(2000, 1, 1, 0, 0, 0)) + vid3.favorite = favorite + test_session.commit() + vid2_video_path = vid2.video_path.path + + favorites = { + simple_channel.link: {vid3.video_path.path.name: {'favorite': favorite}}, + 'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}, + } + + # Save config, verify that favorite is set. + save_channels_config() + config = get_config() + assert config['favorites'] == favorites + + def import_and_verify(favorited_ids: List[int]): + with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'): + import_videos_config() + for video in test_session.query(Video).all(): + if video.id in favorited_ids: + assert video.favorite + else: + assert not video.favorite + + # Clear the favorite (as if the DB was wiped), verify that the favorite is imported and set. + vid2.favorite = None + import_and_verify([vid2.id, vid3.id]) + + # Removing the video does not delete the favorite. If the DB is wiped, we do not want to lose our favorites! + test_session.query(Video).filter_by(id=vid2.id).delete() + test_session.commit() + save_channels_config() + config = get_config() + assert config['favorites'] == favorites + import_and_verify([vid3.id]) + + # Add vid2 again, it should be favorited on import. + vid2 = Video(video_path=vid2_video_path) + test_session.add(vid2) + test_session.commit() + import_and_verify([vid3.id, vid2.id]) + + # Deleting the Video in the model really removes the favorite status. + vid3.delete() + config = get_config() + assert config['favorites'] == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}} + import_and_verify([vid2.id]) diff --git a/wrolpi/common.py b/wrolpi/common.py index 08886e69..0e2c9261 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -233,7 +233,9 @@ def save_settings_config(config=None): if ('channels' in config or 'channels' in local_config) and 'channels' in example_config: del example_config['channels'] - new_config = combine_dicts(config, local_config, example_config) + # Create a new config using the values in order of: config -> local_config -> example_config + all_keys = list(config.keys()) + list(local_config.keys()) + list(example_config.keys()) + new_config = {i: config.get(i, local_config.get(i, example_config.get(i))) for i in all_keys} logger.info(f'Writing config to file: {CONFIG_PATH}') with open(str(CONFIG_PATH), 'wt') as fh: diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 5eae5e86..72bbf183 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -91,11 +91,12 @@ def test_config(): Create a test config based off the example config. """ test_config_path = tempfile.NamedTemporaryFile(mode='rt') - with mock.patch('wrolpi.vars.CONFIG_PATH', test_config_path.name): + with mock.patch('wrolpi.vars.CONFIG_PATH', test_config_path.name), \ + mock.patch('wrolpi.common.CONFIG_PATH', test_config_path.name): config = get_example_config() with open(test_config_path.name, 'wt') as fh: fh.write(yaml.dump(config)) - yield + yield pathlib.Path(test_config_path.name) ROUTES_ATTACHED = False