mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-25 18:40:06 -04:00
03a2df217d
* A new zim module has been added, this module allows several things to interact with Zim files. * Allow searching of Zim files using libzim. * Allow downloading of Kiwix projects (wikipedia, wikibooks, etc) on a schedule which replaces the outdated files. * Creation of a new zim docker container which allows viewing of the Zim file contents. * This commit also adds move/rename/mkdir features in the File Browser. * File tags are preserved when moving/renaming.
489 lines
20 KiB
Python
489 lines
20 KiB
Python
import asyncio
|
|
import shutil
|
|
from copy import copy
|
|
from unittest import mock
|
|
|
|
import pytest
|
|
from yt_dlp.utils import UnsupportedError
|
|
|
|
from modules.videos.channel.lib import download_channel
|
|
from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \
|
|
get_or_create_channel, channel_downloader, video_downloader
|
|
from modules.videos.models import Channel, Video
|
|
from wrolpi.downloader import Download, DownloadResult
|
|
from wrolpi.errors import InvalidDownload
|
|
from wrolpi.test.common import skip_circleci
|
|
from wrolpi.vars import PROJECT_DIR
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_all_missing_videos(test_session, channel_factory, video_factory, image_file):
|
|
channel1 = channel_factory(url='some url')
|
|
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0, 'webpage_url': 'https://example.com'}]}
|
|
test_session.commit()
|
|
|
|
missing = [i async for i in find_all_missing_videos(channel1.id)]
|
|
assert len(missing) == 1
|
|
|
|
# The video returned is the one we faked.
|
|
id_, source_id, entry = missing[0]
|
|
# No Video.id because the video is missing.
|
|
assert id_ is None
|
|
# The fake entry we added is regurgitated back.
|
|
assert entry == channel1.info_json['entries'][0]
|
|
|
|
# Create a video without a poster, attempt to download it.
|
|
video = video_factory(channel_id=channel1.id, with_video_file=True)
|
|
channel1.info_json = {'entries': [
|
|
{'id': 'foo', 'view_count': 0, 'webpage_url': 'https://example.com'},
|
|
{'id': video.source_id, 'view_count': 0, 'webpage_url': 'https://example.com/new'},
|
|
]}
|
|
test_session.commit()
|
|
|
|
missing = [i async for i in find_all_missing_videos(channel1.id)]
|
|
assert len(missing) == 2
|
|
|
|
_, new = missing
|
|
assert new == (video.id, video.source_id, channel1.info_json['entries'][1])
|
|
|
|
|
|
example_video_json = {
|
|
'age_limit': 0,
|
|
'average_rating': 4.6923075,
|
|
'categories': ['Howto & Style'],
|
|
'channel': 'channel name',
|
|
'channel_id': 'channel id',
|
|
'channel_url': 'channel url',
|
|
'duration': 635,
|
|
'extractor': 'youtube',
|
|
'extractor_key': 'Youtube',
|
|
'id': 'some long id',
|
|
'is_live': None,
|
|
'like_count': 24,
|
|
'subtitles': {},
|
|
'title': "The video's title",
|
|
'upload_date': '20190707',
|
|
'uploader': 'uploader name',
|
|
'uploader_id': 'uploader id',
|
|
'uploader_url': 'uploader url',
|
|
'view_count': 406,
|
|
'webpage_url': 'webpage url',
|
|
'webpage_url_basename': 'watch',
|
|
}
|
|
|
|
example_channel_json = {
|
|
'_type': 'playlist',
|
|
'channel_id': 'some id',
|
|
'entries': [
|
|
{'_type': 'url',
|
|
'description': None,
|
|
'duration': None,
|
|
'id': 'video 1 id',
|
|
'ie_key': 'Youtube',
|
|
'title': 'video 1 title',
|
|
'uploader': None,
|
|
'url': 'video_1_url',
|
|
'view_count': 58504,
|
|
'webpage_url': 'https://youtube.com/watch?v=video_1_url'},
|
|
{'_type': 'url',
|
|
'description': None,
|
|
'duration': None,
|
|
'id': 'video 2 id',
|
|
'ie_key': 'Youtube',
|
|
'title': 'video 2 title',
|
|
'uploader': None,
|
|
'url': 'video_2_url',
|
|
'view_count': 1413,
|
|
'webpage_url': 'https://youtube.com/watch?v=video_2_url'},
|
|
],
|
|
'extractor': 'youtube:tab',
|
|
'extractor_key': 'YoutubeTab',
|
|
'id': 'some id',
|
|
'title': 'channel title',
|
|
'uploader': 'the uploader',
|
|
'webpage_url': 'channel url',
|
|
'webpage_url_basename': 'videos',
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_no_channel(test_session, video_download_manager, test_directory,
|
|
mock_video_extract_info, mock_video_process_runner):
|
|
"""A video can be downloaded even if it does not have a Channel."""
|
|
channel_dir = test_directory / 'NO CHANNEL'
|
|
channel_dir.mkdir(parents=True)
|
|
video_path = channel_dir / 'video.mp4'
|
|
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
|
|
|
|
channel_downloader, video_downloader = video_download_manager.instances
|
|
|
|
# Video has no channel
|
|
info_json = copy(example_video_json)
|
|
del info_json['channel']
|
|
del info_json['channel_id']
|
|
del info_json['channel_url']
|
|
|
|
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
|
mock_video_extract_info.return_value = info_json
|
|
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
|
|
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
|
video_download_manager.create_download(url, video_downloader.name)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
|
|
mock_video_process_runner.assert_called_once()
|
|
|
|
video: Video = test_session.query(Video).one()
|
|
assert str(video.video_path) == f'{channel_dir}/video.mp4'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_video_tags(test_session, video_download_manager, video_file_factory, test_directory,
|
|
mock_video_extract_info, mock_video_process_runner, tag_factory):
|
|
"""A Video is tagged when Download record requires it."""
|
|
video_path = video_file_factory()
|
|
tag1, tag2 = tag_factory(), tag_factory()
|
|
|
|
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
|
mock_video_extract_info.return_value = copy(example_video_json)
|
|
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
|
|
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
|
settings = dict(tag_names=[tag1.name, tag2.name])
|
|
video_download_manager.create_download(url, video_downloader.name, settings=settings)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
|
|
mock_video_process_runner.assert_called_once()
|
|
|
|
video: Video = test_session.query(Video).one()
|
|
assert [i.tag.name for i in video.file_group.tag_files] == [tag1.name, tag2.name]
|
|
|
|
|
|
@skip_circleci
|
|
@pytest.mark.skip # TODO this test fails when running multiple tests.
|
|
@pytest.mark.asyncio
|
|
async def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
|
|
mock_video_extract_info, mock_video_prepare_filename,
|
|
mock_video_process_runner):
|
|
"""Downloading (updating the catalog of) a Channel updates it's info_json.
|
|
|
|
If a Channel has `match_regex` only those videos with matching titles will be downloaded."""
|
|
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
|
|
|
|
channel_downloader, video_downloader = video_download_manager.instances
|
|
|
|
mock_video_prepare_filename.return_value = video_file
|
|
mock_video_extract_info.return_value = example_channel_json
|
|
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
|
|
mock_get_channel.return_value = simple_channel
|
|
video_download_manager.create_download(url, channel_downloader.name, sub_downloader_name=video_downloader.name)
|
|
# Download channel.
|
|
await video_download_manager.wait_for_all_downloads()
|
|
# Download videos.
|
|
await video_download_manager.wait_for_all_downloads()
|
|
|
|
# Let background tasks run.
|
|
await asyncio.sleep(0)
|
|
|
|
# Two videos are in the example channel.
|
|
downloads = video_download_manager.get_once_downloads(test_session)
|
|
downloads = filter(lambda i: 'watch' in i.url, downloads)
|
|
assert {i.url for i in downloads} == \
|
|
{'https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url'}
|
|
assert all(i.status == 'complete' for i in downloads)
|
|
|
|
# A channel with `match_regex` only returns matching video URLs.
|
|
simple_channel.match_regex = '.*(2).*'
|
|
test_session.query(Download).delete()
|
|
test_session.commit()
|
|
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
|
|
mock_get_channel.return_value = simple_channel
|
|
video_download_manager.recurring_download(url, 100, channel_downloader.name,
|
|
sub_downloader_name=video_downloader.name)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
downloads = video_download_manager.get_once_downloads(test_session)
|
|
assert [i.url for i in downloads] == ['https://youtube.com/watch?v=video_2_url']
|
|
assert downloads[0].settings == {
|
|
'channel_id': 1,
|
|
'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos',
|
|
}
|
|
|
|
|
|
def test_get_or_create_channel(test_session):
|
|
"""A Channel may need to be created for an arbitrary download.
|
|
|
|
Attempt to use an existing Channel if we can match it.
|
|
"""
|
|
c1 = Channel(name='foo', source_id='foo', url='foo')
|
|
c2 = Channel(name='bar', source_id='bar')
|
|
c3 = Channel(name='baz', source_id='baz', url='baz')
|
|
c4 = Channel(name='qux')
|
|
test_session.add_all([c1, c2, c3, c4])
|
|
test_session.commit()
|
|
|
|
# All existing channels should be used.
|
|
tests = [
|
|
(dict(source_id='foo'), c1),
|
|
(dict(url='foo'), c1),
|
|
(dict(url='foo', source_id='bar'), c2), # source_id has priority.
|
|
(dict(name='foo', source_id='bar'), c2),
|
|
(dict(source_id='bar'), c2),
|
|
(dict(source_id='baz'), c3),
|
|
(dict(name='qux'), c4),
|
|
]
|
|
for kwargs, expected in tests:
|
|
channel = get_or_create_channel(**kwargs)
|
|
assert expected.id == channel.id, f'Expected {expected} for {kwargs} but got {channel}'
|
|
|
|
# A new channel is created. It will not be automatically downloaded.
|
|
channel = get_or_create_channel(source_id='quux', name='quux', url='quux')
|
|
assert channel.id == 5
|
|
assert channel.source_id == 'quux'
|
|
assert channel.name == 'quux'
|
|
assert channel.url == 'quux'
|
|
|
|
# New channel can be retrieved.
|
|
assert get_or_create_channel(source_id='quux') == channel
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_selected_downloader(test_session, video_download_manager, test_downloader, successful_download):
|
|
"""
|
|
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
|
|
"""
|
|
# The test Downloader will pretend to have downloaded anything.
|
|
mock_do_download = test_downloader.do_download = mock.MagicMock()
|
|
mock_do_download.return_value = successful_download
|
|
|
|
def check_attempts(attempts):
|
|
assert test_session.query(Download).one().attempts == attempts
|
|
|
|
# DownloadManager will attempt to choose the Downloader.
|
|
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
|
mock_extract_info.side_effect = UnsupportedError('not this one')
|
|
video_download_manager.create_download('https://example.com', test_downloader.name)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
mock_do_download.assert_called_once()
|
|
|
|
check_attempts(1)
|
|
|
|
mock_do_download.reset_mock()
|
|
|
|
# DownloadManager will only use the TestDownloader.
|
|
video_download_manager.create_download('https://example.com', test_downloader.name)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
# DownloadManager called do_download.
|
|
mock_do_download.assert_called_once()
|
|
|
|
check_attempts(2)
|
|
|
|
mock_do_download.reset_mock()
|
|
|
|
# DownloadManager will only use the TestDownloader, even if the download fails.
|
|
mock_do_download.side_effect = Exception('oh no!')
|
|
video_download_manager.create_download('https://example.com', test_downloader.name)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
# DownloadManager called do_download.
|
|
mock_do_download.assert_called_once()
|
|
|
|
check_attempts(3)
|
|
|
|
|
|
def test_channel_downloader_hidden(video_download_manager):
|
|
"""
|
|
ChannelDownloader should not be presented to the User.
|
|
"""
|
|
downloaders = video_download_manager.list_downloaders()
|
|
assert [i.__json__() for i in downloaders] == [(dict(name='video', pretty_name='Videos')), ]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
def test_bad_downloader(test_session, video_download_manager):
|
|
"""
|
|
Attempting to use an unknown downloader should raise an error.
|
|
"""
|
|
with pytest.raises(InvalidDownload):
|
|
video_download_manager.create_download('https://example.com', downloader_name='bad downloader')
|
|
|
|
|
|
def test_channel_download_no_download(test_session, video_download_manager, simple_channel):
|
|
"""
|
|
A Channel without an existing Download cannot be downloaded. This is to avoid the situation where a single
|
|
video has been downloaded in a Channel, but the User has not requested downloading of all videos in the channel.
|
|
"""
|
|
# simple_channel has no download record.
|
|
assert video_download_manager.get_downloads(test_session) == []
|
|
|
|
with pytest.raises(InvalidDownload):
|
|
download_channel(simple_channel.id)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_download(test_session, test_directory, simple_channel, video_download_manager,
|
|
mock_video_process_runner, image_file):
|
|
"""A video download is performed, files are associated."""
|
|
simple_channel.source_id = example_video_json['channel_id']
|
|
simple_channel.directory = test_directory / 'videos/channel name'
|
|
simple_channel.directory.mkdir(parents=True)
|
|
|
|
video_path = simple_channel.directory / 'a video.mp4'
|
|
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
|
|
# Create a poster file which was downloaded.
|
|
poster_path = video_path.with_suffix('.png')
|
|
image_file.rename(poster_path)
|
|
|
|
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
|
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
|
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
|
|
mock_extract_info.return_value = example_video_json
|
|
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
|
|
|
video_download_manager.create_download(url, video_downloader.name)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
|
|
mock_video_process_runner.assert_called_once()
|
|
video_url, _, out_dir = mock_video_process_runner.call_args[0]
|
|
|
|
download: Download = test_session.query(Download).one()
|
|
assert video_url == download.url
|
|
assert test_session.query(Channel).one()
|
|
|
|
video: Video = test_session.query(Video).one()
|
|
assert video.video_path.is_absolute() and video.video_path == video_path, 'Video path is not absolute'
|
|
assert video.video_path.is_absolute() and video.poster_path == poster_path, 'Video poster was not discovered'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_result(test_session, test_directory, video_download_manager, mock_video_process_runner,
|
|
mock_video_extract_info, image_file):
|
|
"""VideoDownloader returns a DownloadResult when complete."""
|
|
channel_directory = test_directory / 'videos/channel name'
|
|
channel_directory.mkdir(parents=True)
|
|
video_file = channel_directory / 'the video.mp4'
|
|
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file)
|
|
image_file.rename(video_file.with_suffix('.jpg'))
|
|
|
|
mock_video_extract_info.return_value = example_video_json
|
|
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
|
|
mock_prepare_filename.return_value = [video_file, {'id': 'foo'}]
|
|
video_download_manager.create_download('https://example.com', video_downloader.name)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
|
|
download: Download = test_session.query(Download).one()
|
|
assert download.url == 'https://example.com'
|
|
assert download.location == '/videos/channel/1/video/1'
|
|
|
|
# Video has its files.
|
|
video = test_session.query(Video).one()
|
|
assert video.video_path.is_file(), 'Video file was not found.'
|
|
assert video.poster_path.is_file(), 'Poster file was not found.'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_destination(test_session, test_directory, video_download_manager,
|
|
mock_video_process_runner, mock_video_extract_info, mock_video_prepare_filename):
|
|
"""A Video can be downloaded to a directory other than it's Channel's directory."""
|
|
# yt-dlp would return the video json.
|
|
mock_video_extract_info.return_value = example_video_json
|
|
# Download process is successful with no logs.
|
|
mock_video_process_runner.return_value = (0, {'stdout': '', 'stderr': ''})
|
|
# This result will be ignored during the test, we just need some value so the download does not fail.
|
|
mock_video_prepare_filename.return_value = str(test_directory / 'test video.mp4')
|
|
|
|
video_download_manager.create_download('https://example.com/1', downloader_name=VideoDownloader.name)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
# Output directory matches the channel directory.
|
|
assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default'] \
|
|
.startswith(f'{test_directory}/videos/channel name/%(uploader)s')
|
|
|
|
mock_video_prepare_filename.reset_mock()
|
|
|
|
settings = dict(destination=f'{test_directory}/custom')
|
|
video_download_manager.create_download('https://example.com/2', downloader_name=VideoDownloader.name,
|
|
settings=settings)
|
|
await video_download_manager.wait_for_all_downloads()
|
|
# Output directory matches the custom directory specified.
|
|
assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default'] \
|
|
.startswith(f'{test_directory}/custom/%(uploader)s')
|
|
|
|
|
|
example_playlist_json = {
|
|
'_type': 'playlist',
|
|
'availability': None,
|
|
'channel': 'the channel name',
|
|
'channel_follower_count': None,
|
|
'channel_id': 'the channel id',
|
|
'channel_url': 'channel url',
|
|
'description': '',
|
|
'entries': [
|
|
{'_type': 'url',
|
|
'description': None,
|
|
'duration': None,
|
|
'id': 'video 2 id',
|
|
'ie_key': 'Youtube',
|
|
'title': 'video 2 title',
|
|
'uploader': None,
|
|
'url': 'https://www.youtube.com/shorts/video_2_url',
|
|
'view_count': 1413,
|
|
'webpage_url': 'https://www.youtube.com/shorts/video_2_url'},
|
|
{'_type': 'url',
|
|
'description': None,
|
|
'duration': None,
|
|
'id': 'video 1 id',
|
|
'ie_key': 'Youtube',
|
|
'title': 'video 1 title',
|
|
'uploader': None,
|
|
'url': 'https://www.youtube.com/watch?v=video_1_url',
|
|
'view_count': 58504,
|
|
'webpage_url': 'https://www.youtube.com/watch?v=video_1_url'},
|
|
{'_type': 'url',
|
|
'description': None,
|
|
'duration': None,
|
|
'id': 'video 3 id',
|
|
'ie_key': 'Youtube',
|
|
'title': 'video 3 title',
|
|
'uploader': None,
|
|
'url': 'https://youtube.com/watch?v=video_3_url',
|
|
'view_count': 58504,
|
|
'webpage_url': 'https://youtube.com/watch?v=video_3_url'},
|
|
],
|
|
'extractor': 'youtube:tab',
|
|
'extractor_key': 'YoutubeTab',
|
|
'id': 'the playlist id',
|
|
'modified_date': '20220426',
|
|
'original_url': 'original url',
|
|
'playlist_count': 10,
|
|
'requested_entries': None,
|
|
'tags': [],
|
|
'title': 'some title',
|
|
'uploader': 'Playlist Uploader',
|
|
'uploader_id': 'uploader id',
|
|
'uploader_url': 'uploader url',
|
|
'view_count': 22298,
|
|
'webpage_url': 'webpage url',
|
|
'webpage_url_basename': 'playlist',
|
|
'webpage_url_domain': 'youtube.com',
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_playlist(test_session, test_directory, mock_video_extract_info, video_download_manager,
|
|
video_file):
|
|
"""All videos in a playlist can be downloaded for it's Channel."""
|
|
download = Download(url='playlist url')
|
|
test_session.add(download)
|
|
channel = get_or_create_channel(example_playlist_json['channel_id'], download.url, example_channel_json['uploader'])
|
|
video = Video.from_paths(test_session, video_file)
|
|
video.channel_id = channel.id
|
|
video.source_id = 'video 1 id'
|
|
video.url = 'https://www.youtube.com/watch?v=video_1_url'
|
|
test_session.commit()
|
|
|
|
mock_video_extract_info.return_value = example_playlist_json # Playlist info is fetched first.
|
|
|
|
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download:
|
|
mock_video_do_download.return_value = DownloadResult(success=True) # Don't download the videos.
|
|
result = await channel_downloader.do_download(download)
|
|
assert result.success is True, 'Download was not successful'
|
|
assert set(result.downloads) == {
|
|
'https://www.youtube.com/watch?v=video_2_url', # Shorts is converted to regular URL.
|
|
'https://youtube.com/watch?v=video_3_url',
|
|
}
|