mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-25 18:40:06 -04:00
03a2df217d
* A new zim module has been added, this module allows several things to interact with Zim files. * Allow searching of Zim files using libzim. * Allow downloading of Kiwix projects (wikipedia, wikibooks, etc) on a schedule which replaces the outdated files. * Creation of a new zim docker container which allows viewing of the Zim file contents. * This commit also adds move/rename/mkdir features in the File Browser. * File tags are preserved when moving/renaming.
516 lines
21 KiB
Python
516 lines
21 KiB
Python
import json
|
|
import pathlib
|
|
import tempfile
|
|
import time
|
|
from datetime import timedelta
|
|
from http import HTTPStatus
|
|
|
|
import mock
|
|
import pytest
|
|
|
|
from modules.videos.channel.lib import delete_channel
|
|
from modules.videos.models import Channel
|
|
from wrolpi.dates import now
|
|
from wrolpi.db import get_db_session
|
|
from wrolpi.downloader import download_manager, Download, DownloadResult
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
|
|
def test_get_channels(test_directory, channel_factory, test_client):
|
|
channel_factory()
|
|
channel_factory()
|
|
channel_factory()
|
|
request, response = test_client.get('/api/videos/channels')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['channels']) == 3
|
|
|
|
|
|
def test_get_video(test_client, test_session, simple_channel, video_factory):
|
|
"""Test that you get can information about a video. Test that video file can be gotten."""
|
|
now_ = now()
|
|
video1 = video_factory(channel_id=simple_channel.id, title='vid1')
|
|
video1.upload_date = now_
|
|
video2 = video_factory(channel_id=simple_channel.id, title='vid2')
|
|
video2.upload_date = now_ + timedelta(seconds=1)
|
|
test_session.commit()
|
|
|
|
# Test that a 404 is returned when no video exists
|
|
_, response = test_client.get('/api/videos/video/10')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
|
|
assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''}
|
|
|
|
# Get the video info we inserted
|
|
_, response = test_client.get('/api/videos/video/1')
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
assert_dict_contains(response.json['file_group'], {'title': 'vid1'})
|
|
|
|
# The next video is included.
|
|
assert response.json['prev'] is None
|
|
assert_dict_contains(response.json['next'], {'title': 'vid2'})
|
|
|
|
|
|
def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel):
|
|
"""A channel does not require a download frequency."""
|
|
# No downloads are scheduled.
|
|
assert len(download_manager.get_downloads(test_session)) == 0
|
|
|
|
# Get the Channel
|
|
request, response = test_client.get('/api/videos/channels/1')
|
|
channel = response.json['channel']
|
|
assert channel['download_frequency'] is None
|
|
|
|
# Update the Channel with a frequency.
|
|
new_channel = dict(
|
|
directory=channel['directory'],
|
|
name=channel['name'],
|
|
url=channel['url'],
|
|
download_frequency=10,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
# Download is scheduled.
|
|
assert len(download_manager.get_downloads(test_session)) == 1
|
|
|
|
# Remove the frequency.
|
|
new_channel = dict(
|
|
directory=channel['directory'],
|
|
name=channel['name'],
|
|
url=channel['url'],
|
|
download_frequency=None,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
def test_channel_frequency_update(download_channel, test_client, test_session, test_download_manager):
|
|
"""
|
|
A Channel's Download record is updated when the Channel's frequency is updated.
|
|
"""
|
|
old_frequency = download_channel.get_download().frequency
|
|
assert old_frequency
|
|
|
|
data = dict(
|
|
directory=str(download_channel.directory),
|
|
name=download_channel.name,
|
|
url=download_channel.url,
|
|
download_frequency=100,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{download_channel.id}', json=data)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
download = download_channel.get_download()
|
|
assert download.frequency == 100
|
|
|
|
# Only one download
|
|
downloads = test_session.query(Download).all()
|
|
assert len(list(downloads)) == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_channel_download_crud(test_session, simple_channel, test_downloader_config):
|
|
"""Modifying a Channel modifies it's Download."""
|
|
assert simple_channel.url
|
|
assert simple_channel.download_frequency is None
|
|
|
|
simple_channel.update({'download_frequency': 1000})
|
|
test_session.commit()
|
|
|
|
download: Download = test_session.query(Download).one()
|
|
assert simple_channel.get_download() == download
|
|
assert simple_channel.url == download.url
|
|
assert simple_channel.download_frequency == download.frequency
|
|
|
|
# Increase the frequency to 1 second.
|
|
simple_channel.update({'download_frequency': 1})
|
|
test_session.commit()
|
|
assert simple_channel.url == download.url
|
|
assert simple_channel.download_frequency == download.frequency == 1
|
|
assert test_session.query(Download).count() == 1
|
|
|
|
# Changing the Channel's URL changes the download's URL.
|
|
simple_channel.update({'url': 'https://example.com/new url'})
|
|
test_session.commit()
|
|
assert simple_channel.url == 'https://example.com/new url'
|
|
assert simple_channel.download_frequency == download.frequency == 1
|
|
assert test_session.query(Download).count() == 1
|
|
|
|
# Removing the Channel's frequency removes the download.
|
|
simple_channel.update({'download_frequency': None})
|
|
test_session.commit()
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
# Removing the URL is supported.
|
|
simple_channel.update({'url': None})
|
|
test_session.commit()
|
|
|
|
# Adding a URL is supported.
|
|
simple_channel.update({'url': 'https://example.com'})
|
|
test_session.commit()
|
|
|
|
# Adding a frequency adds a Download.
|
|
simple_channel.update({'download_frequency': 1})
|
|
test_session.commit()
|
|
assert test_session.query(Download).count() == 1
|
|
next_download = simple_channel.get_download().next_download
|
|
assert next_download
|
|
|
|
# Next download isn't overwritten.
|
|
time.sleep(1)
|
|
simple_channel.update({'name': 'new name'})
|
|
test_session.commit()
|
|
assert simple_channel.get_download().next_download == next_download
|
|
|
|
# Deleting a Channel deletes it's Download.
|
|
delete_channel(channel_id=simple_channel.id)
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
|
|
def test_video_file_name(test_session, simple_video, test_client):
|
|
"""If a Video has no title, the front-end can use the file name as the title."""
|
|
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
|
|
assert resp.status_code == HTTPStatus.OK
|
|
assert resp.json['file_group']['video']['video_path'] == 'simple_video.mp4'
|
|
assert resp.json['file_group']['video'].get('stem') == 'simple_video'
|
|
|
|
|
|
def test_channel_conflicts(test_client, test_session, test_directory):
|
|
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory,
|
|
match_regex='asdf',
|
|
name='Example Channel 1',
|
|
url='https://example.com/channel1',
|
|
)
|
|
|
|
def _post_channel(channel):
|
|
return test_client.post('/api/videos/channels', content=json.dumps(channel))
|
|
|
|
# Create it
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
# Name is an exact match
|
|
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory2).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory2,
|
|
name='Example Channel 1',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'error': 'The channel name is already taken.', 'code': 5}}
|
|
|
|
# Directory was already used
|
|
new_channel = dict(
|
|
directory=channel_directory,
|
|
name='name is fine',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'code': 7, 'error': 'The directory is already used by another channel.'}}
|
|
|
|
# URL is already used
|
|
channel_directory3 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory3).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory3,
|
|
name='name is fine',
|
|
url='https://example.com/channel1',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'code': 6, 'error': 'The URL is already used by another channel.'}}
|
|
|
|
|
|
def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory):
|
|
"""Two channels with empty URLs shouldn't conflict"""
|
|
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory).mkdir()
|
|
|
|
new_channel = {
|
|
'name': 'Fooz',
|
|
'directory': channel_directory,
|
|
}
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
location = response.headers['Location']
|
|
|
|
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory2).mkdir()
|
|
new_channel = {
|
|
'name': 'Barz',
|
|
'directory': channel_directory2,
|
|
}
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
assert location != response.headers['Location']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_channel_download_requires_refresh(test_session, download_channel, video_download_manager, video_factory,
|
|
events_history):
|
|
"""A Channel cannot be downloaded until it has been refreshed.
|
|
|
|
Videos already downloaded are not downloaded again."""
|
|
vid = video_factory(channel_id=download_channel.id, with_video_file=True, with_poster_ext='jpg')
|
|
vid.url = 'https://example.com/1'
|
|
d = download_channel.get_download()
|
|
test_session.commit()
|
|
assert not d.next_download
|
|
|
|
def assert_refreshed(expected: bool):
|
|
channel: Channel = test_session.query(Channel).one()
|
|
assert channel.refreshed == expected
|
|
if expected:
|
|
assert channel.info_json
|
|
else:
|
|
assert not channel.info_json
|
|
|
|
assert_refreshed(False)
|
|
test_session.commit()
|
|
|
|
downloaded_urls = []
|
|
|
|
async def do_download(_, download):
|
|
# Track all downloaded URLs.
|
|
downloaded_urls.append(download.url)
|
|
return DownloadResult(success=True)
|
|
|
|
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
|
mock.patch('modules.videos.downloader.VideoDownloader.do_download', do_download), \
|
|
mock.patch('wrolpi.files.lib.apply_indexers'): # TODO why does apply_indexers break this test?
|
|
entries = [
|
|
dict(id=vid.source_id, view_count=0, webpage_url='https://example.com/1'), # Already downloaded.
|
|
dict(id='not downloaded', view_count=0, webpage_url='https://example.com/2'),
|
|
]
|
|
mock_extract_info.return_value = {'entries': entries, 'url': 'foo', 'uploader': 'some uploader',
|
|
'channel_id': 'the id', 'id': 'the id'}
|
|
await video_download_manager.do_downloads()
|
|
await video_download_manager.wait_for_all_downloads()
|
|
|
|
# Channel was refreshed before downloading videos.
|
|
assert_refreshed(True)
|
|
assert d.next_download
|
|
# Only the missing video was downloaded.
|
|
assert downloaded_urls == ['https://example.com/2']
|
|
|
|
# Should not send refresh events because downloads are automated.
|
|
assert events_history == []
|
|
|
|
|
|
def test_channel_post_directory(test_session, test_client, test_directory):
|
|
"""A Channel can be created with or without an existing directory."""
|
|
# Channel can be created with a directory which is not on disk.
|
|
data = dict(name='foo', directory='foo')
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
directory = test_session.query(Channel).filter_by(id=1).one().directory
|
|
assert (test_directory / 'foo') == directory
|
|
assert not directory.is_dir()
|
|
assert directory.is_absolute()
|
|
|
|
# Channel can be created and have its directory be created.
|
|
data = dict(name='bar', directory='bar', mkdir=True)
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
directory = test_session.query(Channel).filter_by(id=2).one().directory
|
|
assert (test_directory / 'bar') == directory
|
|
assert directory.is_dir()
|
|
assert directory.is_absolute()
|
|
|
|
|
|
def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
|
|
request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}')
|
|
assert response.status_code == HTTPStatus.OK
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_channel_crud(test_session, test_async_client, test_directory, test_download_manager):
|
|
channel_directory = test_directory / 'channel directory'
|
|
channel_directory.mkdir()
|
|
|
|
new_channel = dict(
|
|
directory=str(channel_directory),
|
|
match_regex='asdf',
|
|
name=' Example Channel 1 ',
|
|
url='https://example.com/channel1',
|
|
)
|
|
|
|
# Channel doesn't exist
|
|
request, response = await test_async_client.get('/api/videos/channels/examplechannel1')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
|
|
|
|
# Create it
|
|
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
location = response.headers['Location']
|
|
|
|
request, response = await test_async_client.get(location)
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
created = response.json['channel']
|
|
assert created
|
|
assert created['id']
|
|
# No frequency was provided. No download exists.
|
|
assert not created['download_frequency']
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
# Channel name leading/trailing whitespace should be stripped
|
|
assert created['name'] == 'Example Channel 1'
|
|
|
|
# Channel directory should be relative to the media directory
|
|
assert not pathlib.Path(created['directory']).is_absolute(), \
|
|
f'Channel directory is absolute: {created["directory"]}'
|
|
|
|
# Can't create it again
|
|
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
async def put_and_fetch(new_channel_):
|
|
request, response = await test_async_client.put(location, content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
|
request, response = await test_async_client.get(location)
|
|
assert response.status_code == HTTPStatus.OK
|
|
channel = response.json['channel']
|
|
return channel
|
|
|
|
# Update it
|
|
new_channel['name'] = 'Example Channel 2'
|
|
new_channel['directory'] = str(new_channel['directory']) # noqa
|
|
new_channel['download_frequency'] = 60
|
|
channel = await put_and_fetch(new_channel)
|
|
assert channel['id'] == 1
|
|
assert channel['name'] == 'Example Channel 2'
|
|
assert channel['directory'] == channel_directory.name
|
|
assert channel['match_regex'] == 'asdf'
|
|
assert channel['url'] == 'https://example.com/channel1'
|
|
# Download was created.
|
|
assert channel['download_frequency']
|
|
assert (downloads := test_download_manager.get_downloads(test_session)) and len(downloads) == 1 and downloads[
|
|
0].url == new_channel['url']
|
|
|
|
# Update with no download frequency.
|
|
new_channel['download_frequency'] = None
|
|
channel = await put_and_fetch(new_channel)
|
|
assert channel['download_frequency'] is None
|
|
# Download was deleted.
|
|
assert len(test_download_manager.get_downloads(test_session)) == 0
|
|
|
|
# Can't update channel that doesn't exist
|
|
request, response = await test_async_client.put('/api/videos/channels/doesnt_exist',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
# Delete the new channel
|
|
request, response = await test_async_client.delete(location)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Cant delete it again
|
|
request, response = await test_async_client.delete(location)
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
|
|
def test_channel_creation_with_download(test_client, test_session, test_directory):
|
|
"""A Download is created when a Channel is created."""
|
|
new_channel = dict(
|
|
directory=str(test_directory),
|
|
name='Download this one',
|
|
url='https://example.com/channel1',
|
|
download_frequency=60,
|
|
)
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
download = test_session.query(Download).one()
|
|
assert download.url == 'https://example.com/channel1'
|
|
assert download.downloader == 'video_channel'
|
|
assert download.sub_downloader == 'video'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_change_channel_url(test_async_client, test_session, test_download_manager, download_channel,
|
|
test_download_manager_config):
|
|
# Change the Channel's URL.
|
|
download_channel.update({'url': 'https://example.com/new-url'})
|
|
test_session.commit()
|
|
|
|
download = test_session.query(Download).one()
|
|
assert download.url == download_channel.url == 'https://example.com/new-url', \
|
|
"Channel's download URL was not changed."
|
|
|
|
download_channel.update({'download_frequency': None})
|
|
assert not list(test_session.query(Download).all())
|
|
|
|
|
|
def test_search_videos_channel(test_client, test_session, video_factory):
|
|
with get_db_session(commit=True) as session:
|
|
channel1 = Channel(name='Foo')
|
|
channel2 = Channel(name='Bar')
|
|
session.add(channel1)
|
|
session.add(channel2)
|
|
session.flush()
|
|
session.refresh(channel1)
|
|
session.refresh(channel2)
|
|
|
|
# Channels don't have videos yet
|
|
d = dict(channel_id=channel1.id)
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['file_groups']) == 0
|
|
|
|
with get_db_session(commit=True) as session:
|
|
vid1 = video_factory(title='vid1', channel_id=channel2.id)
|
|
vid2 = video_factory(title='vid2', channel_id=channel1.id)
|
|
session.add(vid1)
|
|
session.add(vid2)
|
|
|
|
# Videos are gotten by their respective channels
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['file_groups']) == 1
|
|
assert response.json['totals']['file_groups'] == 1
|
|
assert_dict_contains(response.json['file_groups'][0],
|
|
dict(primary_path='vid2.mp4', video=dict(channel_id=channel1.id)))
|
|
|
|
d = dict(channel_id=channel2.id)
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['file_groups']) == 1
|
|
assert_dict_contains(response.json['file_groups'][0],
|
|
dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
|
|
|
|
|
|
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory, assert_video_search):
|
|
for i in range(50):
|
|
video_factory(channel_id=simple_channel.id)
|
|
|
|
channel2 = Channel(name='Bar')
|
|
test_session.add(channel2)
|
|
test_session.flush()
|
|
test_session.refresh(channel2)
|
|
video_factory(channel_id=channel2.id)
|
|
video_factory(title='vid2', channel_id=channel2.id, with_video_file=True)
|
|
test_session.commit()
|
|
|
|
# Get first, second, third, and empty pages of videos.
|
|
tests = [
|
|
# (offset, video_count)
|
|
(0, 20),
|
|
(20, 20),
|
|
(40, 10),
|
|
(50, 0),
|
|
]
|
|
last_ids = []
|
|
for offset, video_count in tests:
|
|
_, response = assert_video_search(channel_id=simple_channel.id, order_by='upload_date', offset=offset, limit=20)
|
|
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
|
|
current_ids = [i['id'] for i in response.json['file_groups']]
|
|
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
|
last_ids = current_ids
|