mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Bugfix/favorites import (#71)
* Favorite videos are saved even if the DB is wiped. * Logging when importing. * Warning when favorite video cannot be found. * Handling when a favorited video has no channel. * Fixing link name in log. * vid2 was not in a channel. Signed-off-by: Roland <roland@learningselfreliance.com>
This commit is contained in:
parent
e6fa9c79f7
commit
bedcc3a5e9
8 changed files with 143 additions and 32 deletions
|
|
@ -11,6 +11,7 @@ from typing import Union, Tuple, List, Set, Iterable, Optional
|
|||
import PIL
|
||||
from PIL import Image
|
||||
from sqlalchemy.orm import Session
|
||||
from yt_dlp import YoutubeDL
|
||||
|
||||
from wrolpi import before_startup
|
||||
from wrolpi.common import logger, get_config, iterify, chunks, get_media_directory, \
|
||||
|
|
@ -58,8 +59,10 @@ class ConfigError(Exception):
|
|||
@before_startup
|
||||
def import_videos_config():
|
||||
"""Import channel settings to the DB. Existing channels will be updated."""
|
||||
logger.info('Importing videos config')
|
||||
try:
|
||||
config = load_channels_config()
|
||||
config = get_config()
|
||||
config, favorites = config['channels'], config.get('favorites', {})
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for link in config:
|
||||
|
|
@ -87,19 +90,35 @@ def import_videos_config():
|
|||
|
||||
if not channel.source_id and channel.url:
|
||||
# If we can download from a channel, we must have its source_id.
|
||||
from .downloader import get_channel_source_id
|
||||
channel.source_id = get_channel_source_id(channel.url)
|
||||
|
||||
session.add(channel)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for link, favorites in favorites.items():
|
||||
if link != 'NO CHANNEL':
|
||||
channel: Channel = session.query(Channel).filter_by(link=link).one_or_none()
|
||||
if not channel:
|
||||
logger.warning(f'Cannot find channel {link=} for favorites!')
|
||||
continue
|
||||
channel_dir = channel.directory.path
|
||||
else:
|
||||
from .downloader import get_no_channel_directory
|
||||
channel_dir = get_no_channel_directory()
|
||||
|
||||
# Set favorite Videos of this Channel.
|
||||
favorites = config[link].get('favorites', {})
|
||||
if favorites:
|
||||
videos = session.query(Video).filter(Video.source_id.in_(favorites.keys()))
|
||||
for video in videos:
|
||||
video.favorite = favorites[video.source_id]['favorite']
|
||||
except Exception:
|
||||
logger.warning('Failed to load channels config!', exc_info=True)
|
||||
for video_path, data in favorites.items():
|
||||
# Favorite in the config is the name of the video_path. Add the channel directory onto this
|
||||
# video_path, so we can match the complete path for the Video.
|
||||
video_path = channel_dir / video_path
|
||||
video = session.query(Video).filter_by(video_path=video_path).one_or_none()
|
||||
# If no Video is found, it may be that we need to refresh.
|
||||
if video:
|
||||
video.favorite = data['favorite']
|
||||
else:
|
||||
logger.warning(f'Cannot find video to favorite: {video_path}')
|
||||
except Exception as e:
|
||||
logger.warning('Failed to load channels config!', exc_info=e)
|
||||
|
||||
|
||||
VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'}
|
||||
|
|
@ -545,3 +564,13 @@ def minimize_video(video: dict) -> dict:
|
|||
video['info_json'] = minimize_video_info_json(video['info_json'])
|
||||
|
||||
return video
|
||||
|
||||
|
||||
YDL = YoutubeDL()
|
||||
YDL.params['logger'] = logger.getChild('youtube-dl')
|
||||
YDL.add_default_info_extractors()
|
||||
|
||||
|
||||
def get_channel_source_id(url: str) -> str:
|
||||
channel_info = YDL.extract_info(url, download=False, process=False)
|
||||
return channel_info.get('uploader_id') or channel_info['id']
|
||||
|
|
|
|||
|
|
@ -101,7 +101,10 @@ def video_file(test_directory) -> pathlib.Path:
|
|||
def video_factory(test_session, test_directory):
|
||||
def _(channel_id: int = None):
|
||||
title = str(uuid4())
|
||||
path = str(test_directory / f'{title}.mp4')
|
||||
if channel_id:
|
||||
path = str(test_directory / f'{title}.mp4')
|
||||
else:
|
||||
path = str(test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4')
|
||||
video = Video(video_path=path, title=title, channel_id=channel_id, source_id=title)
|
||||
test_session.add(video)
|
||||
return video
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from wrolpi.downloader import Downloader, Download
|
|||
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
from .common import load_downloader_config, apply_info_json
|
||||
from .common import load_downloader_config, apply_info_json, get_channel_source_id
|
||||
from .lib import upsert_video, _refresh_videos
|
||||
from .models import Video, Channel
|
||||
from .video_url_resolver import video_url_resolver
|
||||
|
|
@ -469,8 +469,3 @@ def _skip_download(error):
|
|||
if msg in error_str:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_channel_source_id(url: str) -> str:
|
||||
channel_info = YDL.extract_info(url, download=False, process=False)
|
||||
return channel_info.get('uploader_id') or channel_info['id']
|
||||
|
|
|
|||
|
|
@ -3,12 +3,13 @@ import html
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Tuple
|
||||
from uuid import uuid1
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import logger, chunks
|
||||
from wrolpi.common import logger, chunks, get_config
|
||||
from wrolpi.common import save_settings_config
|
||||
from wrolpi.db import get_db_curs, get_db_session
|
||||
from wrolpi.vars import PYTEST
|
||||
|
|
@ -264,10 +265,22 @@ def get_channels_config(session: Session) -> dict:
|
|||
"""
|
||||
channels = session.query(Channel).order_by(Channel.link).all()
|
||||
channels = {i.link: i.config_view() for i in channels}
|
||||
return dict(channels=channels)
|
||||
|
||||
# Get all Videos that are favorites. Store them in their own config section, so they can be preserved if a channel
|
||||
# is deleted or the DB is wiped.
|
||||
favorite_videos = session.query(Video).filter(Video.favorite != None, Video.video_path != None).all() # noqa
|
||||
favorites = defaultdict(lambda: {})
|
||||
for video in favorite_videos:
|
||||
if video.channel:
|
||||
favorites[video.channel.link][video.video_path.path.name] = dict(favorite=video.favorite)
|
||||
else:
|
||||
favorites['NO CHANNEL'][video.video_path.path.name] = dict(favorite=video.favorite)
|
||||
favorites = dict(favorites)
|
||||
|
||||
return dict(channels=channels, favorites=favorites)
|
||||
|
||||
|
||||
def save_channels_config(session=None):
|
||||
def save_channels_config(session=None, preserve_favorites: bool = True):
|
||||
"""
|
||||
Pull the Channel information from the DB, save it to the config.
|
||||
"""
|
||||
|
|
@ -276,6 +289,8 @@ def save_channels_config(session=None):
|
|||
else:
|
||||
with get_db_session() as session:
|
||||
config = get_channels_config(session)
|
||||
if preserve_favorites:
|
||||
config['favorites'].update(get_config().get('favorites', {}))
|
||||
save_settings_config(config)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, Computed
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
|
|
@ -95,7 +95,17 @@ class Video(ModelHelper, Base):
|
|||
if not video_files:
|
||||
raise UnknownFile('No video files were deleted')
|
||||
|
||||
needs_save = bool(self.favorite)
|
||||
|
||||
self.add_to_skip_list()
|
||||
self.favorite = None
|
||||
session = Session.object_session(self)
|
||||
session.commit()
|
||||
|
||||
if needs_save:
|
||||
# Save the Channels config. This video was a favorite and that needs to be removed.
|
||||
from modules.videos.lib import save_channels_config
|
||||
save_channels_config(preserve_favorites=False)
|
||||
|
||||
def add_to_skip_list(self):
|
||||
"""
|
||||
|
|
@ -300,13 +310,6 @@ class Channel(ModelHelper, Base):
|
|||
source_id=self.source_id,
|
||||
url=self.url or '',
|
||||
)
|
||||
|
||||
session = Session.object_session(self)
|
||||
favorites = session.query(Video).filter(Video.favorite != None, Video.channel_id == self.id).all() # noqa
|
||||
favorites: List[Video]
|
||||
if favorites:
|
||||
config['favorites'] = {i.video_path.path.name: {'favorite': i.favorite} for i in favorites}
|
||||
|
||||
return config
|
||||
|
||||
def get_relative_path(self, path: Path, exists: bool = True):
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import json
|
|||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
from unittest import mock
|
||||
from unittest.mock import Mock, MagicMock
|
||||
|
||||
|
|
@ -10,13 +12,15 @@ from PIL import Image
|
|||
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.test.common import create_channel_structure
|
||||
from wrolpi.common import get_absolute_media_path
|
||||
from wrolpi.common import get_absolute_media_path, get_config
|
||||
from wrolpi.dates import local_timezone
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from ..common import get_matching_directories, convert_image, bulk_validate_posters, remove_duplicate_video_paths, \
|
||||
apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \
|
||||
get_bulk_video_info_json
|
||||
get_bulk_video_info_json, import_videos_config
|
||||
from ..lib import save_channels_config
|
||||
|
||||
|
||||
class TestCommon(TestAPI):
|
||||
|
|
@ -405,3 +409,62 @@ async def test_missing_title(test_session, simple_video):
|
|||
test_session.commit()
|
||||
await get_bulk_video_info_json([simple_video.id])
|
||||
assert simple_video.title == 'simple_video'
|
||||
|
||||
|
||||
def test_import_favorites(test_session, simple_channel, video_factory):
|
||||
"""
|
||||
A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be
|
||||
favorited again.
|
||||
|
||||
A Video's favorited status can only be cleared by calling `Video.delete`.
|
||||
"""
|
||||
video_factory(channel_id=simple_channel.id) # never favorited
|
||||
vid2 = video_factory(channel_id=None) # has no channel
|
||||
vid3 = video_factory(channel_id=simple_channel.id)
|
||||
favorite = vid2.favorite = local_timezone(datetime(2000, 1, 1, 0, 0, 0))
|
||||
vid3.favorite = favorite
|
||||
test_session.commit()
|
||||
vid2_video_path = vid2.video_path.path
|
||||
|
||||
favorites = {
|
||||
simple_channel.link: {vid3.video_path.path.name: {'favorite': favorite}},
|
||||
'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}},
|
||||
}
|
||||
|
||||
# Save config, verify that favorite is set.
|
||||
save_channels_config()
|
||||
config = get_config()
|
||||
assert config['favorites'] == favorites
|
||||
|
||||
def import_and_verify(favorited_ids: List[int]):
|
||||
with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'):
|
||||
import_videos_config()
|
||||
for video in test_session.query(Video).all():
|
||||
if video.id in favorited_ids:
|
||||
assert video.favorite
|
||||
else:
|
||||
assert not video.favorite
|
||||
|
||||
# Clear the favorite (as if the DB was wiped), verify that the favorite is imported and set.
|
||||
vid2.favorite = None
|
||||
import_and_verify([vid2.id, vid3.id])
|
||||
|
||||
# Removing the video does not delete the favorite. If the DB is wiped, we do not want to lose our favorites!
|
||||
test_session.query(Video).filter_by(id=vid2.id).delete()
|
||||
test_session.commit()
|
||||
save_channels_config()
|
||||
config = get_config()
|
||||
assert config['favorites'] == favorites
|
||||
import_and_verify([vid3.id])
|
||||
|
||||
# Add vid2 again, it should be favorited on import.
|
||||
vid2 = Video(video_path=vid2_video_path)
|
||||
test_session.add(vid2)
|
||||
test_session.commit()
|
||||
import_and_verify([vid3.id, vid2.id])
|
||||
|
||||
# Deleting the Video in the model really removes the favorite status.
|
||||
vid3.delete()
|
||||
config = get_config()
|
||||
assert config['favorites'] == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}}
|
||||
import_and_verify([vid2.id])
|
||||
|
|
|
|||
|
|
@ -233,7 +233,9 @@ def save_settings_config(config=None):
|
|||
if ('channels' in config or 'channels' in local_config) and 'channels' in example_config:
|
||||
del example_config['channels']
|
||||
|
||||
new_config = combine_dicts(config, local_config, example_config)
|
||||
# Create a new config using the values in order of: config -> local_config -> example_config
|
||||
all_keys = list(config.keys()) + list(local_config.keys()) + list(example_config.keys())
|
||||
new_config = {i: config.get(i, local_config.get(i, example_config.get(i))) for i in all_keys}
|
||||
|
||||
logger.info(f'Writing config to file: {CONFIG_PATH}')
|
||||
with open(str(CONFIG_PATH), 'wt') as fh:
|
||||
|
|
|
|||
|
|
@ -91,11 +91,12 @@ def test_config():
|
|||
Create a test config based off the example config.
|
||||
"""
|
||||
test_config_path = tempfile.NamedTemporaryFile(mode='rt')
|
||||
with mock.patch('wrolpi.vars.CONFIG_PATH', test_config_path.name):
|
||||
with mock.patch('wrolpi.vars.CONFIG_PATH', test_config_path.name), \
|
||||
mock.patch('wrolpi.common.CONFIG_PATH', test_config_path.name):
|
||||
config = get_example_config()
|
||||
with open(test_config_path.name, 'wt') as fh:
|
||||
fh.write(yaml.dump(config))
|
||||
yield
|
||||
yield pathlib.Path(test_config_path.name)
|
||||
|
||||
|
||||
ROUTES_ATTACHED = False
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue