mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 23:46:18 -05:00
515 lines
20 KiB
Python
515 lines
20 KiB
Python
import json
|
|
import pathlib
|
|
import tempfile
|
|
import time
|
|
from datetime import timedelta
|
|
from http import HTTPStatus
|
|
|
|
import mock
|
|
|
|
from modules.videos.channel.lib import delete_channel
|
|
from modules.videos.common import get_no_channel_directory
|
|
from modules.videos.models import Channel, Video
|
|
from wrolpi.dates import now
|
|
from wrolpi.db import get_db_session
|
|
from wrolpi.downloader import download_manager, Download
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
|
|
def test_get_channels(test_directory, channel_factory, test_client):
|
|
channel_factory()
|
|
channel_factory()
|
|
channel_factory()
|
|
request, response = test_client.get('/api/videos/channels')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['channels']) == 3
|
|
|
|
|
|
@mock.patch('modules.videos.lib.validate_video')
|
|
def test_refresh_no_channel(_, test_session, test_client, test_directory, simple_channel):
|
|
# A regular user-defined channel
|
|
vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
|
|
vid1.touch()
|
|
vid2 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title2.mp4'
|
|
vid2.touch()
|
|
|
|
# The special NO CHANNEL directory.
|
|
no_channel_path = get_no_channel_directory()
|
|
vid3 = no_channel_path / 'some video.mp4'
|
|
vid3.touch()
|
|
no_channel_subdir = no_channel_path / 'subdir'
|
|
no_channel_subdir.mkdir()
|
|
vid4 = no_channel_subdir / 'other video.mp4'
|
|
vid4.touch()
|
|
|
|
test_client.post('/api/videos/refresh')
|
|
assert test_session.query(Video).count() == 4
|
|
|
|
|
|
def test_get_video(test_client, test_session, simple_channel, video_factory):
|
|
"""
|
|
Test that you get can information about a video. Test that video file can be gotten.
|
|
"""
|
|
now_ = now()
|
|
video1 = video_factory(channel_id=simple_channel.id)
|
|
video1.upload_date, video1.title = now_, 'vid1'
|
|
video2 = video_factory(channel_id=simple_channel.id)
|
|
video2.upload_date, video2.title = now_ + timedelta(seconds=1), 'vid2'
|
|
test_session.commit()
|
|
|
|
# Test that a 404 is returned when no video exists
|
|
_, response = test_client.get('/api/videos/video/10')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
|
|
assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''}
|
|
|
|
# Get the video info we inserted
|
|
_, response = test_client.get('/api/videos/video/1')
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
assert_dict_contains(response.json['video'], {'title': 'vid1'})
|
|
|
|
# The next video is included.
|
|
assert response.json['prev'] is None
|
|
assert_dict_contains(response.json['next'], {'title': 'vid2'})
|
|
|
|
|
|
def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel):
|
|
"""A channel does not require a download frequency."""
|
|
# No downloads are scheduled.
|
|
assert len(download_manager.get_downloads(test_session)) == 0
|
|
|
|
# Get the Channel
|
|
request, response = test_client.get('/api/videos/channels/1')
|
|
channel = response.json['channel']
|
|
assert channel['download_frequency'] is None
|
|
|
|
# Update the Channel with a frequency.
|
|
new_channel = dict(
|
|
directory=channel['directory'],
|
|
name=channel['name'],
|
|
url=channel['url'],
|
|
download_frequency=10,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
# Download is scheduled.
|
|
assert len(download_manager.get_downloads(test_session)) == 1
|
|
|
|
# Remove the frequency.
|
|
new_channel = dict(
|
|
directory=channel['directory'],
|
|
name=channel['name'],
|
|
url=channel['url'],
|
|
download_frequency=None,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
|
|
def test_channel_frequency_update(download_channel, test_client, test_session):
|
|
"""
|
|
A Channel's Download record is updated when the Channel's frequency is updated.
|
|
"""
|
|
old_frequency = download_channel.get_download().frequency
|
|
assert old_frequency
|
|
|
|
data = dict(
|
|
directory=str(download_channel.directory.path),
|
|
name=download_channel.name,
|
|
url=download_channel.url,
|
|
download_frequency=100,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{download_channel.id}', json=data)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
download = download_channel.get_download()
|
|
assert download.frequency == 100
|
|
|
|
# Only one download
|
|
downloads = test_session.query(Download).all()
|
|
assert len(list(downloads)) == 1
|
|
|
|
|
|
def test_channel_download_crud(test_session, simple_channel):
|
|
"""Modifying a Channel modifies it's Download."""
|
|
assert simple_channel.url
|
|
assert simple_channel.download_frequency is None
|
|
|
|
simple_channel.update({'download_frequency': 1000})
|
|
test_session.commit()
|
|
|
|
download: Download = test_session.query(Download).one()
|
|
assert simple_channel.get_download() == download
|
|
assert simple_channel.url == download.url
|
|
assert simple_channel.download_frequency == download.frequency
|
|
|
|
# Increase the frequency to 1 second.
|
|
simple_channel.update({'download_frequency': 1})
|
|
test_session.commit()
|
|
assert simple_channel.url == download.url
|
|
assert simple_channel.download_frequency == download.frequency == 1
|
|
assert test_session.query(Download).count() == 1
|
|
|
|
# Changing the Channel's URL changes the download's URL.
|
|
simple_channel.update({'url': 'https://example.com/new url'})
|
|
test_session.commit()
|
|
assert simple_channel.url == 'https://example.com/new url'
|
|
assert simple_channel.download_frequency == download.frequency == 1
|
|
assert test_session.query(Download).count() == 1
|
|
|
|
# Removing the Channel's frequency removes the download.
|
|
simple_channel.update({'download_frequency': None})
|
|
test_session.commit()
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
# Removing the URL is supported.
|
|
simple_channel.update({'url': None})
|
|
test_session.commit()
|
|
|
|
# Adding a URL is supported.
|
|
simple_channel.update({'url': 'https://example.com'})
|
|
test_session.commit()
|
|
|
|
# Adding a frequency adds a Download.
|
|
simple_channel.update({'download_frequency': 1})
|
|
test_session.commit()
|
|
assert test_session.query(Download).count() == 1
|
|
next_download = simple_channel.get_download().next_download
|
|
assert next_download
|
|
|
|
# Next download isn't overwritten.
|
|
time.sleep(1)
|
|
simple_channel.update({'name': 'new name'})
|
|
test_session.commit()
|
|
assert simple_channel.get_download().next_download == next_download
|
|
|
|
# Deleting a Channel deletes it's Download.
|
|
delete_channel(channel_id=simple_channel.id)
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
|
|
def test_video_search(test_client, test_session):
|
|
"""
|
|
Test that videos can be searched and that their order is by their textsearch rank.
|
|
"""
|
|
# These captions have repeated letters, so they will be higher in the ranking.
|
|
videos = [
|
|
('1', 'b b b b e d d'),
|
|
('2', '2 b b b d'),
|
|
('3', 'b b'),
|
|
('4', 'b e e'),
|
|
('5', ''),
|
|
]
|
|
for title, caption in videos:
|
|
test_session.add(Video(title=title, caption=caption, video_path='foo'))
|
|
test_session.commit()
|
|
|
|
def do_search(search_str, limit=20):
|
|
d = json.dumps({'search_str': search_str, 'limit': limit})
|
|
_, resp = test_client.post('/api/videos/search', content=d)
|
|
return resp
|
|
|
|
def search_is_as_expected(resp, expected):
|
|
assert resp.status_code == HTTPStatus.OK
|
|
response_ids = [i['id'] for i in resp.json['videos']]
|
|
assert response_ids == expected
|
|
assert resp.json['totals']['videos'] == len(expected)
|
|
|
|
# Repeated runs should return the same result
|
|
for _ in range(2):
|
|
# Only videos with a b are returned, ordered by the amount of b's
|
|
response = do_search('b')
|
|
search_is_as_expected(response, [1, 2, 3, 4])
|
|
|
|
# Only two captions have e
|
|
response = do_search('e')
|
|
search_is_as_expected(response, [4, 1])
|
|
|
|
# Only two captions have d
|
|
response = do_search('d')
|
|
search_is_as_expected(response, [1, 2])
|
|
|
|
# 5 can be gotten by it's title
|
|
response = do_search('5')
|
|
search_is_as_expected(response, [5])
|
|
|
|
# only video 1 has e and d
|
|
response = do_search('e d')
|
|
search_is_as_expected(response, [1])
|
|
|
|
# video 1 and 4 have b and e, but 1 has more
|
|
response = do_search('b e')
|
|
search_is_as_expected(response, [1, 4])
|
|
|
|
# Check totals are correct even with a limit
|
|
response = do_search('b', 2)
|
|
assert [i['id'] for i in response.json['videos']] == [1, 2]
|
|
assert response.json['totals']['videos'] == 4
|
|
|
|
|
|
def test_video_file_name(test_session, simple_video, test_client):
|
|
"""
|
|
If a Video has no title, the front-end can use the file name as the title.
|
|
"""
|
|
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
|
|
assert resp.status_code == HTTPStatus.OK
|
|
assert resp.json['video']['video_path'] == 'simple_video.mp4'
|
|
assert resp.json['video'].get('stem') == 'simple_video'
|
|
|
|
|
|
def test_channel_conflicts(test_client, test_session, test_directory):
|
|
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory,
|
|
match_regex='asdf',
|
|
name='Example Channel 1',
|
|
url='https://example.com/channel1',
|
|
)
|
|
|
|
def _post_channel(channel):
|
|
return test_client.post('/api/videos/channels', content=json.dumps(channel))
|
|
|
|
# Create it
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
# Name is an exact match
|
|
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory2).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory2,
|
|
name='Example Channel 1',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'error': 'The channel name is already taken.', 'code': 5}}
|
|
|
|
# Directory was already used
|
|
new_channel = dict(
|
|
directory=channel_directory,
|
|
name='name is fine',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'code': 7, 'error': 'The directory is already used by another channel.'}}
|
|
|
|
# URL is already used
|
|
channel_directory3 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory3).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory3,
|
|
name='name is fine',
|
|
url='https://example.com/channel1',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'code': 6, 'error': 'The URL is already used by another channel.'}}
|
|
|
|
|
|
def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory):
|
|
"""Two channels with empty URLs shouldn't conflict"""
|
|
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory).mkdir()
|
|
|
|
new_channel = {
|
|
'name': 'Fooz',
|
|
'directory': channel_directory,
|
|
}
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
location = response.headers['Location']
|
|
|
|
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory2).mkdir()
|
|
new_channel = {
|
|
'name': 'Barz',
|
|
'directory': channel_directory2,
|
|
}
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
assert location != response.headers['Location']
|
|
|
|
|
|
def test_download_channel_no_refresh(test_session, download_channel, video_download_manager):
|
|
"""A Channel cannot be downloaded until it has been refreshed."""
|
|
|
|
def check_refreshed(expected: bool):
|
|
channel = test_session.query(Channel).one()
|
|
assert channel.refreshed == expected
|
|
|
|
check_refreshed(False)
|
|
test_session.commit()
|
|
|
|
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
|
mock_extract_info.return_value = {'entries': [], 'url': 'foo', 'uploader': 'some uploader',
|
|
'channel_id': 'the id', 'id': 'the id'}
|
|
video_download_manager.do_downloads_sync()
|
|
|
|
check_refreshed(True)
|
|
|
|
|
|
def test_channel_post_directory(test_session, test_client, test_directory):
|
|
"""A Channel can be created with or without an existing directory."""
|
|
# Channel can be created with a directory which is not on disk.
|
|
data = dict(name='foo', directory='foo')
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
directory = test_session.query(Channel).filter_by(id=1).one().directory.path
|
|
assert (test_directory / 'foo') == directory
|
|
assert not directory.is_dir()
|
|
assert directory.is_absolute()
|
|
|
|
# Channel can be created and have its directory be created.
|
|
data = dict(name='bar', directory='bar', mkdir=True)
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
directory = test_session.query(Channel).filter_by(id=2).one().directory.path
|
|
assert (test_directory / 'bar') == directory
|
|
assert directory.is_dir()
|
|
assert directory.is_absolute()
|
|
|
|
|
|
def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
|
|
request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}')
|
|
assert response.status_code == HTTPStatus.OK
|
|
|
|
|
|
def test_channel_crud(test_session, test_client, test_directory):
|
|
channel_directory = test_directory / 'channel directory'
|
|
channel_directory.mkdir()
|
|
|
|
new_channel = dict(
|
|
directory=str(channel_directory),
|
|
match_regex='asdf',
|
|
name=' Example Channel 1 ',
|
|
url='https://example.com/channel1',
|
|
)
|
|
|
|
# Channel doesn't exist
|
|
request, response = test_client.get('/api/videos/channels/examplechannel1')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
|
|
|
|
# Create it
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
location = response.headers['Location']
|
|
|
|
request, response = test_client.get(location)
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
created = response.json['channel']
|
|
assert created
|
|
assert created['id']
|
|
# No frequency was provided. No download exists.
|
|
assert not created['download_frequency']
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
# Channel name leading/trailing whitespace should be stripped
|
|
assert created['name'] == 'Example Channel 1'
|
|
|
|
# Channel directory should be relative to the media directory
|
|
assert not pathlib.Path(created['directory']).is_absolute(), \
|
|
f'Channel directory is absolute: {created["directory"]}'
|
|
|
|
# Can't create it again
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
# Update it
|
|
new_channel['name'] = 'Example Channel 2'
|
|
new_channel['directory'] = str(new_channel['directory']) # noqa
|
|
request, response = test_client.put(location, content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
|
request, response = test_client.get(location)
|
|
assert response.status_code == HTTPStatus.OK
|
|
channel = response.json['channel']
|
|
assert channel['id'] == 1
|
|
assert channel['name'] == 'Example Channel 2'
|
|
assert channel['directory'] == channel_directory.name
|
|
assert channel['match_regex'] == 'asdf'
|
|
assert channel['url'] == 'https://example.com/channel1'
|
|
|
|
# Can't update channel that doesn't exist
|
|
request, response = test_client.put('/api/videos/channels/doesnt_exist',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
# Delete the new channel
|
|
request, response = test_client.delete(location)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Cant delete it again
|
|
request, response = test_client.delete(location)
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
|
|
def test_search_channel_videos(test_client, test_session):
|
|
with get_db_session(commit=True) as session:
|
|
channel1 = Channel(name='Foo')
|
|
channel2 = Channel(name='Bar')
|
|
session.add(channel1)
|
|
session.add(channel2)
|
|
session.flush()
|
|
session.refresh(channel1)
|
|
session.refresh(channel2)
|
|
|
|
# Channels don't have videos yet
|
|
d = dict(channel_id=channel1.id)
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['videos']) == 0
|
|
|
|
with get_db_session(commit=True) as session:
|
|
vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo')
|
|
vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo')
|
|
session.add(vid1)
|
|
session.add(vid2)
|
|
|
|
# Videos are gotten by their respective channels
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['videos']) == 1
|
|
assert response.json['totals']['videos'] == 1
|
|
assert_dict_contains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id))
|
|
|
|
d = dict(channel_id=channel2.id)
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['videos']) == 1
|
|
assert_dict_contains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id))
|
|
|
|
|
|
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory):
|
|
for i in range(50):
|
|
video_factory(channel_id=simple_channel.id)
|
|
|
|
channel2 = Channel(name='Bar')
|
|
test_session.add(channel2)
|
|
test_session.flush()
|
|
test_session.refresh(channel2)
|
|
video_factory(channel_id=channel2.id)
|
|
test_session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo'))
|
|
test_session.commit()
|
|
|
|
# Get first, second, third, and empty pages of videos.
|
|
tests = [
|
|
# (offset, video_count)
|
|
(0, 20),
|
|
(20, 20),
|
|
(40, 10),
|
|
(50, 0),
|
|
]
|
|
last_ids = []
|
|
for offset, video_count in tests:
|
|
d = dict(channel_id=simple_channel.id, order_by='id', offset=offset)
|
|
_, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['videos']) == video_count
|
|
current_ids = [i['id'] for i in response.json['videos']]
|
|
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
|
last_ids = current_ids
|