mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Removing default download freqeuncy so channels will not be downloaded without a url and frequency. Adding channel unique url constraint. Handling when a Channel's directory may still be relative in DB. Adding TODOs to remove old favorites. Removing outdated favorites from config after favorites have been created. Testing that old-style favorited videos in the config are imported. Removing channels_by_link. Using list to store channels in config. Moving channel import function into lib. Supporting old channel import with links. Bumping version for channel.link removal. Removing all reference to Channel.link in app and api. Adding relative property to MediaPathType. Testing that a channel can be downloaded after it has a frequency added. All tests pass. Video and Channel tests pass without channel.link. Removing channel.link usage in channel api. Changing video Statistics page to About.
523 lines
20 KiB
Python
523 lines
20 KiB
Python
import json
|
|
import pathlib
|
|
import tempfile
|
|
from datetime import timedelta
|
|
from http import HTTPStatus
|
|
|
|
import mock
|
|
|
|
from modules.videos.channel.lib import delete_channel
|
|
from modules.videos.common import get_no_channel_directory
|
|
from modules.videos.models import Channel, Video
|
|
from wrolpi.dates import now
|
|
from wrolpi.db import get_db_session
|
|
from wrolpi.downloader import download_manager, Download
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
|
|
def test_get_channels(test_directory, channel_factory, test_client):
|
|
channel_factory()
|
|
channel_factory()
|
|
channel_factory()
|
|
request, response = test_client.get('/api/videos/channels')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['channels']) == 3
|
|
|
|
|
|
def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel):
|
|
# A regular user-defined channel
|
|
vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
|
|
vid1.touch()
|
|
vid2 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title2.mp4'
|
|
vid2.touch()
|
|
|
|
# The special NO CHANNEL directory.
|
|
no_channel_path = get_no_channel_directory()
|
|
vid3 = no_channel_path / 'some video.mp4'
|
|
vid3.touch()
|
|
no_channel_subdir = no_channel_path / 'subdir'
|
|
no_channel_subdir.mkdir()
|
|
vid4 = no_channel_subdir / 'other video.mp4'
|
|
vid4.touch()
|
|
|
|
test_client.post('/api/videos/refresh')
|
|
|
|
videos = test_session.query(Video).order_by(Video.title).all()
|
|
assert len(videos) == 4
|
|
|
|
# Add some fake videos, they should be removed.
|
|
test_session.add(Video(title='vid5'))
|
|
test_session.add(Video(title='vid6', channel_id=simple_channel.id))
|
|
test_session.commit()
|
|
|
|
videos = test_session.query(Video).order_by(Video.title).all()
|
|
assert len(videos) == 6
|
|
|
|
test_client.post('/api/videos/refresh')
|
|
|
|
videos = test_session.query(Video).order_by(Video.title).all()
|
|
assert len(videos) == 4
|
|
|
|
|
|
def test_get_video(test_client, test_session, simple_channel, video_factory):
|
|
"""
|
|
Test that you get can information about a video. Test that video file can be gotten.
|
|
"""
|
|
now_ = now()
|
|
video1 = video_factory(channel_id=simple_channel.id)
|
|
video1.upload_date, video1.title = now_, 'vid1'
|
|
video2 = video_factory(channel_id=simple_channel.id)
|
|
video2.upload_date, video2.title = now_ + timedelta(seconds=1), 'vid2'
|
|
test_session.commit()
|
|
|
|
# Test that a 404 is returned when no video exists
|
|
_, response = test_client.get('/api/videos/video/10')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
|
|
assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''}
|
|
|
|
# Get the video info we inserted
|
|
_, response = test_client.get('/api/videos/video/1')
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
assert_dict_contains(response.json['video'], {'title': 'vid1'})
|
|
|
|
# The next video is included.
|
|
assert response.json['prev'] is None
|
|
assert_dict_contains(response.json['next'], {'title': 'vid2'})
|
|
|
|
|
|
def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel):
|
|
"""A channel does not require a download frequency."""
|
|
# No downloads are scheduled.
|
|
assert len(download_manager.get_downloads(test_session)) == 0
|
|
|
|
# Get the Channel
|
|
request, response = test_client.get('/api/videos/channels/1')
|
|
channel = response.json['channel']
|
|
assert channel['download_frequency'] is None
|
|
|
|
# Update the Channel with a frequency.
|
|
new_channel = dict(
|
|
directory=channel['directory'],
|
|
name=channel['name'],
|
|
url=channel['url'],
|
|
download_frequency=10,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
# Download is scheduled.
|
|
assert len(download_manager.get_downloads(test_session)) == 1
|
|
|
|
# Remove the frequency.
|
|
new_channel = dict(
|
|
directory=channel['directory'],
|
|
name=channel['name'],
|
|
url=channel['url'],
|
|
download_frequency=None,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
|
|
def test_channel_frequency_update(download_channel, test_client, test_session):
|
|
"""
|
|
A Channel's Download record is updated when the Channel's frequency is updated.
|
|
"""
|
|
old_frequency = download_channel.get_download().frequency
|
|
assert old_frequency
|
|
|
|
data = dict(
|
|
directory=str(download_channel.directory.path),
|
|
name=download_channel.name,
|
|
url=download_channel.url,
|
|
download_frequency=100,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{download_channel.id}', json=data)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
download = download_channel.get_download()
|
|
assert download.frequency == 100
|
|
|
|
# Only one download
|
|
downloads = test_session.query(Download).all()
|
|
assert len(list(downloads)) == 1
|
|
|
|
|
|
def test_channel_download_crud(test_session, simple_channel):
|
|
"""Modifying a Channel modifies it's Download."""
|
|
assert simple_channel.url
|
|
assert simple_channel.download_frequency is None
|
|
|
|
simple_channel.update({'download_frequency': 1000})
|
|
test_session.commit()
|
|
|
|
download: Download = test_session.query(Download).one()
|
|
assert simple_channel.get_download() == download
|
|
assert simple_channel.url == download.url
|
|
assert simple_channel.download_frequency == download.frequency
|
|
|
|
# Increase the frequency to 1 second.
|
|
simple_channel.update({'download_frequency': 1})
|
|
test_session.commit()
|
|
assert simple_channel.url == download.url
|
|
assert simple_channel.download_frequency == download.frequency == 1
|
|
assert test_session.query(Download).count() == 1
|
|
|
|
# Changing the Channel's URL changes the download's URL.
|
|
simple_channel.update({'url': 'https://example.com/new url'})
|
|
test_session.commit()
|
|
assert simple_channel.url == 'https://example.com/new url'
|
|
assert simple_channel.download_frequency == download.frequency == 1
|
|
assert test_session.query(Download).count() == 1
|
|
|
|
# Removing the Channel's frequency removes the download.
|
|
simple_channel.update({'download_frequency': None})
|
|
test_session.commit()
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
# Removing the URL is supported.
|
|
simple_channel.update({'url': None})
|
|
test_session.commit()
|
|
|
|
# Adding a URL is supported.
|
|
simple_channel.update({'url': 'https://example.com'})
|
|
test_session.commit()
|
|
|
|
|
|
def test_delete_channel_delete_download(download_channel, test_session):
|
|
"""Deleting a Channel deletes it's Download."""
|
|
downloads = test_session.query(Download).all()
|
|
assert downloads and all(i.url == download_channel.url for i in downloads)
|
|
|
|
delete_channel(channel_id=download_channel.id)
|
|
|
|
downloads = test_session.query(Download).all()
|
|
assert not list(downloads)
|
|
|
|
|
|
def test_video_search(test_client, test_session):
|
|
"""
|
|
Test that videos can be searched and that their order is by their textsearch rank.
|
|
"""
|
|
# These captions have repeated letters, so they will be higher in the ranking.
|
|
videos = [
|
|
('1', 'b b b b e d d'),
|
|
('2', '2 b b b d'),
|
|
('3', 'b b'),
|
|
('4', 'b e e'),
|
|
('5', ''),
|
|
]
|
|
for title, caption in videos:
|
|
test_session.add(Video(title=title, caption=caption, video_path='foo'))
|
|
test_session.commit()
|
|
|
|
def do_search(search_str, limit=20):
|
|
d = json.dumps({'search_str': search_str, 'limit': limit})
|
|
_, resp = test_client.post('/api/videos/search', content=d)
|
|
return resp
|
|
|
|
def search_is_as_expected(resp, expected):
|
|
assert resp.status_code == HTTPStatus.OK
|
|
response_ids = [i['id'] for i in resp.json['videos']]
|
|
assert response_ids == expected
|
|
assert resp.json['totals']['videos'] == len(expected)
|
|
|
|
# Repeated runs should return the same result
|
|
for _ in range(2):
|
|
# Only videos with a b are returned, ordered by the amount of b's
|
|
response = do_search('b')
|
|
search_is_as_expected(response, [1, 2, 3, 4])
|
|
|
|
# Only two captions have e
|
|
response = do_search('e')
|
|
search_is_as_expected(response, [4, 1])
|
|
|
|
# Only two captions have d
|
|
response = do_search('d')
|
|
search_is_as_expected(response, [1, 2])
|
|
|
|
# 5 can be gotten by it's title
|
|
response = do_search('5')
|
|
search_is_as_expected(response, [5])
|
|
|
|
# only video 1 has e and d
|
|
response = do_search('e d')
|
|
search_is_as_expected(response, [1])
|
|
|
|
# video 1 and 4 have b and e, but 1 has more
|
|
response = do_search('b e')
|
|
search_is_as_expected(response, [1, 4])
|
|
|
|
# Check totals are correct even with a limit
|
|
response = do_search('b', 2)
|
|
assert [i['id'] for i in response.json['videos']] == [1, 2]
|
|
assert response.json['totals']['videos'] == 4
|
|
|
|
|
|
def test_video_file_name(test_session, simple_video, test_client):
|
|
"""
|
|
If a Video has no title, the front-end can use the file name as the title.
|
|
"""
|
|
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
|
|
assert resp.status_code == HTTPStatus.OK
|
|
assert resp.json['video']['video_path'] == 'simple_video.mp4'
|
|
assert resp.json['video'].get('stem') == 'simple_video'
|
|
|
|
|
|
def test_channel_conflicts(test_client, test_session, test_directory):
|
|
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory,
|
|
match_regex='asdf',
|
|
name='Example Channel 1',
|
|
url='https://example.com/channel1',
|
|
)
|
|
|
|
def _post_channel(channel):
|
|
return test_client.post('/api/videos/channels', content=json.dumps(channel))
|
|
|
|
# Create it
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
# Name is an exact match
|
|
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory2).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory2,
|
|
name='Example Channel 1',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'error': 'The channel name is already taken.', 'code': 5}}
|
|
|
|
# Directory was already used
|
|
new_channel = dict(
|
|
directory=channel_directory,
|
|
name='name is fine',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'code': 7, 'error': 'The directory is already used by another channel.'}}
|
|
|
|
# URL is already used
|
|
channel_directory3 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory3).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory3,
|
|
name='name is fine',
|
|
url='https://example.com/channel1',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'code': 6, 'error': 'The URL is already used by another channel.'}}
|
|
|
|
|
|
def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory):
|
|
"""Two channels with empty URLs shouldn't conflict"""
|
|
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory).mkdir()
|
|
|
|
new_channel = {
|
|
'name': 'Fooz',
|
|
'directory': channel_directory,
|
|
}
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
location = response.headers['Location']
|
|
|
|
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory2).mkdir()
|
|
new_channel = {
|
|
'name': 'Barz',
|
|
'directory': channel_directory2,
|
|
}
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
assert location != response.headers['Location']
|
|
|
|
|
|
def test_download_channel_no_refresh(test_session, download_channel, video_download_manager):
|
|
"""
|
|
A Channel cannot be downloaded until it has been refreshed.
|
|
"""
|
|
|
|
def check_refreshed(expected: bool):
|
|
channel = test_session.query(Channel).one()
|
|
assert channel.refreshed == expected
|
|
|
|
check_refreshed(False)
|
|
test_session.commit()
|
|
|
|
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
|
mock_extract_info.return_value = {'entries': [], 'url': 'foo'}
|
|
video_download_manager.do_downloads_sync()
|
|
|
|
check_refreshed(True)
|
|
|
|
|
|
def test_channel_post_directory(test_session, test_client, test_directory):
|
|
"""A Channel can be created with or without an existing directory."""
|
|
# Channel can be created with a directory which is not on disk.
|
|
data = dict(name='foo', directory='foo')
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
directory = test_session.query(Channel).filter_by(id=1).one().directory.path
|
|
assert (test_directory / 'foo') == directory
|
|
assert not directory.is_dir()
|
|
assert directory.is_absolute()
|
|
|
|
# Channel can be created and have its directory be created.
|
|
data = dict(name='bar', directory='bar', mkdir=True)
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
directory = test_session.query(Channel).filter_by(id=2).one().directory.path
|
|
assert (test_directory / 'bar') == directory
|
|
assert directory.is_dir()
|
|
assert directory.is_absolute()
|
|
|
|
|
|
def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
|
|
request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}')
|
|
assert response.status_code == HTTPStatus.OK
|
|
|
|
|
|
def test_channel_crud(test_session, test_client, test_directory):
|
|
channel_directory = test_directory / 'channel directory'
|
|
channel_directory.mkdir()
|
|
|
|
new_channel = dict(
|
|
directory=str(channel_directory),
|
|
match_regex='asdf',
|
|
name=' Example Channel 1 ',
|
|
url='https://example.com/channel1',
|
|
)
|
|
|
|
# Channel doesn't exist
|
|
request, response = test_client.get('/api/videos/channels/examplechannel1')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
|
|
|
|
# Create it
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
location = response.headers['Location']
|
|
|
|
request, response = test_client.get(location)
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
created = response.json['channel']
|
|
assert created
|
|
assert created['id']
|
|
# No frequency was provided. No download exists.
|
|
assert not created['download_frequency']
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
# Channel name leading/trailing whitespace should be stripped
|
|
assert created['name'] == 'Example Channel 1'
|
|
|
|
# Channel directory should be relative to the media directory
|
|
assert not pathlib.Path(created['directory']).is_absolute(), \
|
|
f'Channel directory is absolute: {created["directory"]}'
|
|
|
|
# Can't create it again
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
# Update it
|
|
new_channel['name'] = 'Example Channel 2'
|
|
new_channel['directory'] = str(new_channel['directory']) # noqa
|
|
request, response = test_client.put(location, content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
|
request, response = test_client.get(location)
|
|
assert response.status_code == HTTPStatus.OK
|
|
channel = response.json['channel']
|
|
assert channel['id'] == 1
|
|
assert channel['name'] == 'Example Channel 2'
|
|
assert channel['directory'] == channel_directory.name
|
|
assert channel['match_regex'] == 'asdf'
|
|
assert channel['url'] == 'https://example.com/channel1'
|
|
|
|
# Can't update channel that doesn't exist
|
|
request, response = test_client.put('/api/videos/channels/doesnt_exist',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
# Delete the new channel
|
|
request, response = test_client.delete(location)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Cant delete it again
|
|
request, response = test_client.delete(location)
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
|
|
def test_search_channel_videos(test_client, test_session):
|
|
with get_db_session(commit=True) as session:
|
|
channel1 = Channel(name='Foo')
|
|
channel2 = Channel(name='Bar')
|
|
session.add(channel1)
|
|
session.add(channel2)
|
|
session.flush()
|
|
session.refresh(channel1)
|
|
session.refresh(channel2)
|
|
|
|
# Channels don't have videos yet
|
|
d = dict(channel_id=channel1.id)
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['videos']) == 0
|
|
|
|
with get_db_session(commit=True) as session:
|
|
vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo')
|
|
vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo')
|
|
session.add(vid1)
|
|
session.add(vid2)
|
|
|
|
# Videos are gotten by their respective channels
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['videos']) == 1
|
|
assert response.json['totals']['videos'] == 1
|
|
assert_dict_contains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id))
|
|
|
|
d = dict(channel_id=channel2.id)
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['videos']) == 1
|
|
assert_dict_contains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id))
|
|
|
|
|
|
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory):
|
|
for i in range(50):
|
|
video_factory(channel_id=simple_channel.id)
|
|
|
|
channel2 = Channel(name='Bar')
|
|
test_session.add(channel2)
|
|
test_session.flush()
|
|
test_session.refresh(channel2)
|
|
video_factory(channel_id=channel2.id)
|
|
test_session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo'))
|
|
test_session.commit()
|
|
|
|
# Get first, second, third, and empty pages of videos.
|
|
tests = [
|
|
# (offset, video_count)
|
|
(0, 20),
|
|
(20, 20),
|
|
(40, 10),
|
|
(50, 0),
|
|
]
|
|
last_ids = []
|
|
for offset, video_count in tests:
|
|
d = dict(channel_id=simple_channel.id, order_by='id', offset=offset)
|
|
_, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['videos']) == video_count
|
|
current_ids = [i['id'] for i in response.json['videos']]
|
|
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
|
last_ids = current_ids
|