Bugfix/favorites import (#71)

* Favorite videos are saved even if the DB is wiped.

* Logging when importing.

* Warning when favorite video cannot be found.

* Handling when a favorited video has no channel.

* Fixing link name in log.

* vid2 was not in a channel.

Signed-off-by: Roland <roland@learningselfreliance.com>
This commit is contained in:
lrnselfreliance 2022-01-05 12:52:23 -07:00 • committed by Roland
parent e6fa9c79f7
commit bedcc3a5e9
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
8 changed files with 143 additions and 32 deletions

View file

@ -11,6 +11,7 @@ from typing import Union, Tuple, List, Set, Iterable, Optional
import PIL
from PIL import Image
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from wrolpi import before_startup
from wrolpi.common import logger, get_config, iterify, chunks, get_media_directory, \
@ -58,8 +59,10 @@ class ConfigError(Exception):
@before_startup
def import_videos_config():
"""Import channel settings to the DB. Existing channels will be updated."""
logger.info('Importing videos config')
try:
config = load_channels_config()
config = get_config()
config, favorites = config['channels'], config.get('favorites', {})
with get_db_session(commit=True) as session:
for link in config:
@ -87,19 +90,35 @@ def import_videos_config():
if not channel.source_id and channel.url:
# If we can download from a channel, we must have its source_id.
from .downloader import get_channel_source_id
channel.source_id = get_channel_source_id(channel.url)
session.add(channel)
with get_db_session(commit=True) as session:
for link, favorites in favorites.items():
if link != 'NO CHANNEL':
channel: Channel = session.query(Channel).filter_by(link=link).one_or_none()
if not channel:
logger.warning(f'Cannot find channel {link=} for favorites!')
continue
channel_dir = channel.directory.path
else:
from .downloader import get_no_channel_directory
channel_dir = get_no_channel_directory()
# Set favorite Videos of this Channel.
favorites = config[link].get('favorites', {})
if favorites:
videos = session.query(Video).filter(Video.source_id.in_(favorites.keys()))
for video in videos:
video.favorite = favorites[video.source_id]['favorite']
except Exception:
logger.warning('Failed to load channels config!', exc_info=True)
for video_path, data in favorites.items():
# Favorite in the config is the name of the video_path. Add the channel directory onto this
# video_path, so we can match the complete path for the Video.
video_path = channel_dir / video_path
video = session.query(Video).filter_by(video_path=video_path).one_or_none()
# If no Video is found, it may be that we need to refresh.
if video:
video.favorite = data['favorite']
else:
logger.warning(f'Cannot find video to favorite: {video_path}')
except Exception as e:
logger.warning('Failed to load channels config!', exc_info=e)
VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'}
@ -545,3 +564,13 @@ def minimize_video(video: dict) -> dict:
video['info_json'] = minimize_video_info_json(video['info_json'])
return video
YDL = YoutubeDL()
YDL.params['logger'] = logger.getChild('youtube-dl')
YDL.add_default_info_extractors()
def get_channel_source_id(url: str) -> str:
channel_info = YDL.extract_info(url, download=False, process=False)
return channel_info.get('uploader_id') or channel_info['id']

View file

@ -101,7 +101,10 @@ def video_file(test_directory) -> pathlib.Path:
def video_factory(test_session, test_directory):
def _(channel_id: int = None):
title = str(uuid4())
path = str(test_directory / f'{title}.mp4')
if channel_id:
path = str(test_directory / f'{title}.mp4')
else:
path = str(test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4')
video = Video(video_path=path, title=title, channel_id=channel_id, source_id=title)
test_session.add(video)
return video

View file

@ -19,7 +19,7 @@ from wrolpi.downloader import Downloader, Download
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import load_downloader_config, apply_info_json
from .common import load_downloader_config, apply_info_json, get_channel_source_id
from .lib import upsert_video, _refresh_videos
from .models import Video, Channel
from .video_url_resolver import video_url_resolver
@ -469,8 +469,3 @@ def _skip_download(error):
if msg in error_str:
return True
return False
def get_channel_source_id(url: str) -> str:
channel_info = YDL.extract_info(url, download=False, process=False)
return channel_info.get('uploader_id') or channel_info['id']

View file

@ -3,12 +3,13 @@ import html
import json
import pathlib
import re
from collections import defaultdict
from typing import Tuple
from uuid import uuid1
from sqlalchemy.orm import Session
from wrolpi.common import logger, chunks
from wrolpi.common import logger, chunks, get_config
from wrolpi.common import save_settings_config
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.vars import PYTEST
@ -264,10 +265,22 @@ def get_channels_config(session: Session) -> dict:
"""
channels = session.query(Channel).order_by(Channel.link).all()
channels = {i.link: i.config_view() for i in channels}
return dict(channels=channels)
# Get all Videos that are favorites. Store them in their own config section, so they can be preserved if a channel
# is deleted or the DB is wiped.
favorite_videos = session.query(Video).filter(Video.favorite != None, Video.video_path != None).all() # noqa
favorites = defaultdict(lambda: {})
for video in favorite_videos:
if video.channel:
favorites[video.channel.link][video.video_path.path.name] = dict(favorite=video.favorite)
else:
favorites['NO CHANNEL'][video.video_path.path.name] = dict(favorite=video.favorite)
favorites = dict(favorites)
return dict(channels=channels, favorites=favorites)
def save_channels_config(session=None):
def save_channels_config(session=None, preserve_favorites: bool = True):
"""
Pull the Channel information from the DB, save it to the config.
"""
@ -276,6 +289,8 @@ def save_channels_config(session=None):
else:
with get_db_session() as session:
config = get_channels_config(session)
if preserve_favorites:
config['favorites'].update(get_config().get('favorites', {}))
save_settings_config(config)

View file

@ -1,7 +1,7 @@
import json
from datetime import datetime
from pathlib import Path
from typing import Optional, List
from typing import Optional
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, Computed
from sqlalchemy.orm import relationship, Session
@ -95,7 +95,17 @@ class Video(ModelHelper, Base):
if not video_files:
raise UnknownFile('No video files were deleted')
needs_save = bool(self.favorite)
self.add_to_skip_list()
self.favorite = None
session = Session.object_session(self)
session.commit()
if needs_save:
# Save the Channels config. This video was a favorite and that needs to be removed.
from modules.videos.lib import save_channels_config
save_channels_config(preserve_favorites=False)
def add_to_skip_list(self):
"""
@ -300,13 +310,6 @@ class Channel(ModelHelper, Base):
source_id=self.source_id,
url=self.url or '',
)
session = Session.object_session(self)
favorites = session.query(Video).filter(Video.favorite != None, Video.channel_id == self.id).all() # noqa
favorites: List[Video]
if favorites:
config['favorites'] = {i.video_path.path.name: {'favorite': i.favorite} for i in favorites}
return config
def get_relative_path(self, path: Path, exists: bool = True):

View file

@ -2,6 +2,8 @@ import json
import pathlib
import subprocess
import tempfile
from datetime import datetime
from typing import List
from unittest import mock
from unittest.mock import Mock, MagicMock
@ -10,13 +12,15 @@ from PIL import Image
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from wrolpi.common import get_absolute_media_path
from wrolpi.common import get_absolute_media_path, get_config
from wrolpi.dates import local_timezone
from wrolpi.db import get_db_session
from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
from wrolpi.vars import PROJECT_DIR
from ..common import get_matching_directories, convert_image, bulk_validate_posters, remove_duplicate_video_paths, \
apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \
get_bulk_video_info_json
get_bulk_video_info_json, import_videos_config
from ..lib import save_channels_config
class TestCommon(TestAPI):
@ -405,3 +409,62 @@ async def test_missing_title(test_session, simple_video):
test_session.commit()
await get_bulk_video_info_json([simple_video.id])
assert simple_video.title == 'simple_video'
def test_import_favorites(test_session, simple_channel, video_factory):
"""
A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be
favorited again.
A Video's favorited status can only be cleared by calling `Video.delete`.
"""
video_factory(channel_id=simple_channel.id) # never favorited
vid2 = video_factory(channel_id=None) # has no channel
vid3 = video_factory(channel_id=simple_channel.id)
favorite = vid2.favorite = local_timezone(datetime(2000, 1, 1, 0, 0, 0))
vid3.favorite = favorite
test_session.commit()
vid2_video_path = vid2.video_path.path
favorites = {
simple_channel.link: {vid3.video_path.path.name: {'favorite': favorite}},
'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}},
}
# Save config, verify that favorite is set.
save_channels_config()
config = get_config()
assert config['favorites'] == favorites
def import_and_verify(favorited_ids: List[int]):
with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'):
import_videos_config()
for video in test_session.query(Video).all():
if video.id in favorited_ids:
assert video.favorite
else:
assert not video.favorite
# Clear the favorite (as if the DB was wiped), verify that the favorite is imported and set.
vid2.favorite = None
import_and_verify([vid2.id, vid3.id])
# Removing the video does not delete the favorite. If the DB is wiped, we do not want to lose our favorites!
test_session.query(Video).filter_by(id=vid2.id).delete()
test_session.commit()
save_channels_config()
config = get_config()
assert config['favorites'] == favorites
import_and_verify([vid3.id])
# Add vid2 again, it should be favorited on import.
vid2 = Video(video_path=vid2_video_path)
test_session.add(vid2)
test_session.commit()
import_and_verify([vid3.id, vid2.id])
# Deleting the Video in the model really removes the favorite status.
vid3.delete()
config = get_config()
assert config['favorites'] == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}}
import_and_verify([vid2.id])

View file

@ -233,7 +233,9 @@ def save_settings_config(config=None):
if ('channels' in config or 'channels' in local_config) and 'channels' in example_config:
del example_config['channels']
new_config = combine_dicts(config, local_config, example_config)
# Create a new config using the values in order of: config -> local_config -> example_config
all_keys = list(config.keys()) + list(local_config.keys()) + list(example_config.keys())
new_config = {i: config.get(i, local_config.get(i, example_config.get(i))) for i in all_keys}
logger.info(f'Writing config to file: {CONFIG_PATH}')
with open(str(CONFIG_PATH), 'wt') as fh:

View file

@ -91,11 +91,12 @@ def test_config():
Create a test config based off the example config.
"""
test_config_path = tempfile.NamedTemporaryFile(mode='rt')
with mock.patch('wrolpi.vars.CONFIG_PATH', test_config_path.name):
with mock.patch('wrolpi.vars.CONFIG_PATH', test_config_path.name), \
mock.patch('wrolpi.common.CONFIG_PATH', test_config_path.name):
config = get_example_config()
with open(test_config_path.name, 'wt') as fh:
fh.write(yaml.dump(config))
yield
yield pathlib.Path(test_config_path.name)
ROUTES_ATTACHED = False