wrolpi/modules/videos/channel/test/test_api.py
Roland aa203cc102
Moving non-video functions to wrolpi.common. Handling Channel/Downlaod relationship in Channel.update.
Removing default download freqeuncy so channels will not be downloaded without a url and frequency.

Adding channel unique url constraint.

Handling when a Channel's directory may still be relative in DB.

Adding TODOs to remove old favorites.

Removing outdated favorites from config after favorites have been created.

Testing that old-style favorited videos in the config are imported.

Removing channels_by_link.

Using list to store channels in config.

Moving channel import function into lib.

Supporting old channel import with links.

Bumping version for channel.link removal.

Removing all reference to Channel.link in app and api.

Adding relative property to MediaPathType.

Testing that a channel can be downloaded after it has a frequency added.

All tests pass.

Video and Channel tests pass without channel.link.

Removing channel.link usage in channel api.

Changing video Statistics page to About.
2022-04-29 19:09:25 -06:00

523 lines
20 KiB
Python

import json
import pathlib
import tempfile
from datetime import timedelta
from http import HTTPStatus
import mock
from modules.videos.channel.lib import delete_channel
from modules.videos.common import get_no_channel_directory
from modules.videos.models import Channel, Video
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import download_manager, Download
from wrolpi.test.common import assert_dict_contains
def test_get_channels(test_directory, channel_factory, test_client):
channel_factory()
channel_factory()
channel_factory()
request, response = test_client.get('/api/videos/channels')
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 3
def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel):
# A regular user-defined channel
vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
vid1.touch()
vid2 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title2.mp4'
vid2.touch()
# The special NO CHANNEL directory.
no_channel_path = get_no_channel_directory()
vid3 = no_channel_path / 'some video.mp4'
vid3.touch()
no_channel_subdir = no_channel_path / 'subdir'
no_channel_subdir.mkdir()
vid4 = no_channel_subdir / 'other video.mp4'
vid4.touch()
test_client.post('/api/videos/refresh')
videos = test_session.query(Video).order_by(Video.title).all()
assert len(videos) == 4
# Add some fake videos, they should be removed.
test_session.add(Video(title='vid5'))
test_session.add(Video(title='vid6', channel_id=simple_channel.id))
test_session.commit()
videos = test_session.query(Video).order_by(Video.title).all()
assert len(videos) == 6
test_client.post('/api/videos/refresh')
videos = test_session.query(Video).order_by(Video.title).all()
assert len(videos) == 4
def test_get_video(test_client, test_session, simple_channel, video_factory):
"""
Test that you get can information about a video. Test that video file can be gotten.
"""
now_ = now()
video1 = video_factory(channel_id=simple_channel.id)
video1.upload_date, video1.title = now_, 'vid1'
video2 = video_factory(channel_id=simple_channel.id)
video2.upload_date, video2.title = now_ + timedelta(seconds=1), 'vid2'
test_session.commit()
# Test that a 404 is returned when no video exists
_, response = test_client.get('/api/videos/video/10')
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''}
# Get the video info we inserted
_, response = test_client.get('/api/videos/video/1')
assert response.status_code == HTTPStatus.OK, response.json
assert_dict_contains(response.json['video'], {'title': 'vid1'})
# The next video is included.
assert response.json['prev'] is None
assert_dict_contains(response.json['next'], {'title': 'vid2'})
def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel):
"""A channel does not require a download frequency."""
# No downloads are scheduled.
assert len(download_manager.get_downloads(test_session)) == 0
# Get the Channel
request, response = test_client.get('/api/videos/channels/1')
channel = response.json['channel']
assert channel['download_frequency'] is None
# Update the Channel with a frequency.
new_channel = dict(
directory=channel['directory'],
name=channel['name'],
url=channel['url'],
download_frequency=10,
)
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
# Download is scheduled.
assert len(download_manager.get_downloads(test_session)) == 1
# Remove the frequency.
new_channel = dict(
directory=channel['directory'],
name=channel['name'],
url=channel['url'],
download_frequency=None,
)
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
def test_channel_frequency_update(download_channel, test_client, test_session):
"""
A Channel's Download record is updated when the Channel's frequency is updated.
"""
old_frequency = download_channel.get_download().frequency
assert old_frequency
data = dict(
directory=str(download_channel.directory.path),
name=download_channel.name,
url=download_channel.url,
download_frequency=100,
)
request, response = test_client.put(f'/api/videos/channels/{download_channel.id}', json=data)
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
download = download_channel.get_download()
assert download.frequency == 100
# Only one download
downloads = test_session.query(Download).all()
assert len(list(downloads)) == 1
def test_channel_download_crud(test_session, simple_channel):
"""Modifying a Channel modifies it's Download."""
assert simple_channel.url
assert simple_channel.download_frequency is None
simple_channel.update({'download_frequency': 1000})
test_session.commit()
download: Download = test_session.query(Download).one()
assert simple_channel.get_download() == download
assert simple_channel.url == download.url
assert simple_channel.download_frequency == download.frequency
# Increase the frequency to 1 second.
simple_channel.update({'download_frequency': 1})
test_session.commit()
assert simple_channel.url == download.url
assert simple_channel.download_frequency == download.frequency == 1
assert test_session.query(Download).count() == 1
# Changing the Channel's URL changes the download's URL.
simple_channel.update({'url': 'https://example.com/new url'})
test_session.commit()
assert simple_channel.url == 'https://example.com/new url'
assert simple_channel.download_frequency == download.frequency == 1
assert test_session.query(Download).count() == 1
# Removing the Channel's frequency removes the download.
simple_channel.update({'download_frequency': None})
test_session.commit()
assert test_session.query(Download).count() == 0
# Removing the URL is supported.
simple_channel.update({'url': None})
test_session.commit()
# Adding a URL is supported.
simple_channel.update({'url': 'https://example.com'})
test_session.commit()
def test_delete_channel_delete_download(download_channel, test_session):
"""Deleting a Channel deletes it's Download."""
downloads = test_session.query(Download).all()
assert downloads and all(i.url == download_channel.url for i in downloads)
delete_channel(channel_id=download_channel.id)
downloads = test_session.query(Download).all()
assert not list(downloads)
def test_video_search(test_client, test_session):
"""
Test that videos can be searched and that their order is by their textsearch rank.
"""
# These captions have repeated letters, so they will be higher in the ranking.
videos = [
('1', 'b b b b e d d'),
('2', '2 b b b d'),
('3', 'b b'),
('4', 'b e e'),
('5', ''),
]
for title, caption in videos:
test_session.add(Video(title=title, caption=caption, video_path='foo'))
test_session.commit()
def do_search(search_str, limit=20):
d = json.dumps({'search_str': search_str, 'limit': limit})
_, resp = test_client.post('/api/videos/search', content=d)
return resp
def search_is_as_expected(resp, expected):
assert resp.status_code == HTTPStatus.OK
response_ids = [i['id'] for i in resp.json['videos']]
assert response_ids == expected
assert resp.json['totals']['videos'] == len(expected)
# Repeated runs should return the same result
for _ in range(2):
# Only videos with a b are returned, ordered by the amount of b's
response = do_search('b')
search_is_as_expected(response, [1, 2, 3, 4])
# Only two captions have e
response = do_search('e')
search_is_as_expected(response, [4, 1])
# Only two captions have d
response = do_search('d')
search_is_as_expected(response, [1, 2])
# 5 can be gotten by it's title
response = do_search('5')
search_is_as_expected(response, [5])
# only video 1 has e and d
response = do_search('e d')
search_is_as_expected(response, [1])
# video 1 and 4 have b and e, but 1 has more
response = do_search('b e')
search_is_as_expected(response, [1, 4])
# Check totals are correct even with a limit
response = do_search('b', 2)
assert [i['id'] for i in response.json['videos']] == [1, 2]
assert response.json['totals']['videos'] == 4
def test_video_file_name(test_session, simple_video, test_client):
"""
If a Video has no title, the front-end can use the file name as the title.
"""
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
assert resp.status_code == HTTPStatus.OK
assert resp.json['video']['video_path'] == 'simple_video.mp4'
assert resp.json['video'].get('stem') == 'simple_video'
def test_channel_conflicts(test_client, test_session, test_directory):
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
new_channel = dict(
directory=channel_directory,
match_regex='asdf',
name='Example Channel 1',
url='https://example.com/channel1',
)
def _post_channel(channel):
return test_client.post('/api/videos/channels', content=json.dumps(channel))
# Create it
request, response = _post_channel(new_channel)
assert response.status_code == HTTPStatus.CREATED
# Name is an exact match
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory2).mkdir()
new_channel = dict(
directory=channel_directory2,
name='Example Channel 1',
)
request, response = _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
'cause': {'error': 'The channel name is already taken.', 'code': 5}}
# Directory was already used
new_channel = dict(
directory=channel_directory,
name='name is fine',
)
request, response = _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
'cause': {'code': 7, 'error': 'The directory is already used by another channel.'}}
# URL is already used
channel_directory3 = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory3).mkdir()
new_channel = dict(
directory=channel_directory3,
name='name is fine',
url='https://example.com/channel1',
)
request, response = _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
'cause': {'code': 6, 'error': 'The URL is already used by another channel.'}}
def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory):
"""Two channels with empty URLs shouldn't conflict"""
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
new_channel = {
'name': 'Fooz',
'directory': channel_directory,
}
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED, response.json
location = response.headers['Location']
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory2).mkdir()
new_channel = {
'name': 'Barz',
'directory': channel_directory2,
}
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED, response.json
assert location != response.headers['Location']
def test_download_channel_no_refresh(test_session, download_channel, video_download_manager):
"""
A Channel cannot be downloaded until it has been refreshed.
"""
def check_refreshed(expected: bool):
channel = test_session.query(Channel).one()
assert channel.refreshed == expected
check_refreshed(False)
test_session.commit()
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_extract_info.return_value = {'entries': [], 'url': 'foo'}
video_download_manager.do_downloads_sync()
check_refreshed(True)
def test_channel_post_directory(test_session, test_client, test_directory):
"""A Channel can be created with or without an existing directory."""
# Channel can be created with a directory which is not on disk.
data = dict(name='foo', directory='foo')
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
assert response.status_code == HTTPStatus.CREATED
directory = test_session.query(Channel).filter_by(id=1).one().directory.path
assert (test_directory / 'foo') == directory
assert not directory.is_dir()
assert directory.is_absolute()
# Channel can be created and have its directory be created.
data = dict(name='bar', directory='bar', mkdir=True)
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
assert response.status_code == HTTPStatus.CREATED
directory = test_session.query(Channel).filter_by(id=2).one().directory.path
assert (test_directory / 'bar') == directory
assert directory.is_dir()
assert directory.is_absolute()
def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}')
assert response.status_code == HTTPStatus.OK
def test_channel_crud(test_session, test_client, test_directory):
channel_directory = test_directory / 'channel directory'
channel_directory.mkdir()
new_channel = dict(
directory=str(channel_directory),
match_regex='asdf',
name=' Example Channel 1 ',
url='https://example.com/channel1',
)
# Channel doesn't exist
request, response = test_client.get('/api/videos/channels/examplechannel1')
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
# Create it
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED, response.json
location = response.headers['Location']
request, response = test_client.get(location)
assert response.status_code == HTTPStatus.OK, response.json
created = response.json['channel']
assert created
assert created['id']
# No frequency was provided. No download exists.
assert not created['download_frequency']
assert test_session.query(Download).count() == 0
# Channel name leading/trailing whitespace should be stripped
assert created['name'] == 'Example Channel 1'
# Channel directory should be relative to the media directory
assert not pathlib.Path(created['directory']).is_absolute(), \
f'Channel directory is absolute: {created["directory"]}'
# Can't create it again
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.BAD_REQUEST
# Update it
new_channel['name'] = 'Example Channel 2'
new_channel['directory'] = str(new_channel['directory']) # noqa
request, response = test_client.put(location, content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = test_client.get(location)
assert response.status_code == HTTPStatus.OK
channel = response.json['channel']
assert channel['id'] == 1
assert channel['name'] == 'Example Channel 2'
assert channel['directory'] == channel_directory.name
assert channel['match_regex'] == 'asdf'
assert channel['url'] == 'https://example.com/channel1'
# Can't update channel that doesn't exist
request, response = test_client.put('/api/videos/channels/doesnt_exist',
content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NOT_FOUND
# Delete the new channel
request, response = test_client.delete(location)
assert response.status_code == HTTPStatus.NO_CONTENT
# Cant delete it again
request, response = test_client.delete(location)
assert response.status_code == HTTPStatus.NOT_FOUND
def test_search_channel_videos(test_client, test_session):
with get_db_session(commit=True) as session:
channel1 = Channel(name='Foo')
channel2 = Channel(name='Bar')
session.add(channel1)
session.add(channel2)
session.flush()
session.refresh(channel1)
session.refresh(channel2)
# Channels don't have videos yet
d = dict(channel_id=channel1.id)
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 0
with get_db_session(commit=True) as session:
vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo')
vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo')
session.add(vid1)
session.add(vid2)
# Videos are gotten by their respective channels
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 1
assert response.json['totals']['videos'] == 1
assert_dict_contains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id))
d = dict(channel_id=channel2.id)
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 1
assert_dict_contains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id))
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory):
for i in range(50):
video_factory(channel_id=simple_channel.id)
channel2 = Channel(name='Bar')
test_session.add(channel2)
test_session.flush()
test_session.refresh(channel2)
video_factory(channel_id=channel2.id)
test_session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo'))
test_session.commit()
# Get first, second, third, and empty pages of videos.
tests = [
# (offset, video_count)
(0, 20),
(20, 20),
(40, 10),
(50, 0),
]
last_ids = []
for offset, video_count in tests:
d = dict(channel_id=simple_channel.id, order_by='id', offset=offset)
_, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == video_count
current_ids = [i['id'] for i in response.json['videos']]
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
last_ids = current_ids