mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
494 lines
20 KiB
Python
494 lines
20 KiB
Python
import json
|
|
import pathlib
|
|
import tempfile
|
|
import time
|
|
from datetime import timedelta
|
|
from http import HTTPStatus
|
|
|
|
import mock
|
|
import pytest
|
|
|
|
from modules.videos.channel.lib import delete_channel
|
|
from modules.videos.models import Channel
|
|
from wrolpi.dates import now
|
|
from wrolpi.db import get_db_session
|
|
from wrolpi.downloader import download_manager, Download, DownloadResult
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
|
|
def test_get_channels(test_directory, channel_factory, test_client):
|
|
channel_factory()
|
|
channel_factory()
|
|
channel_factory()
|
|
request, response = test_client.get('/api/videos/channels')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['channels']) == 3
|
|
|
|
|
|
def test_get_video(test_client, test_session, simple_channel, video_factory):
|
|
"""Test that you get can information about a video. Test that video file can be gotten."""
|
|
now_ = now()
|
|
video1 = video_factory(channel_id=simple_channel.id, title='vid1')
|
|
video1.upload_date = now_
|
|
video2 = video_factory(channel_id=simple_channel.id, title='vid2')
|
|
video2.upload_date = now_ + timedelta(seconds=1)
|
|
test_session.commit()
|
|
|
|
# Test that a 404 is returned when no video exists
|
|
_, response = test_client.get('/api/videos/video/10')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
|
|
assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''}
|
|
|
|
# Get the video info we inserted
|
|
_, response = test_client.get('/api/videos/video/1')
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
assert_dict_contains(response.json['file_group'], {'title': 'vid1'})
|
|
|
|
# The next video is included.
|
|
assert response.json['prev'] is None
|
|
assert_dict_contains(response.json['next'], {'title': 'vid2'})
|
|
|
|
|
|
def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel):
|
|
"""A channel does not require a download frequency."""
|
|
# No downloads are scheduled.
|
|
assert len(download_manager.get_downloads(test_session)) == 0
|
|
|
|
# Get the Channel
|
|
request, response = test_client.get('/api/videos/channels/1')
|
|
channel = response.json['channel']
|
|
assert channel['download_frequency'] is None
|
|
|
|
# Update the Channel with a frequency.
|
|
new_channel = dict(
|
|
directory=channel['directory'],
|
|
name=channel['name'],
|
|
url=channel['url'],
|
|
download_frequency=10,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
# Download is scheduled.
|
|
assert len(download_manager.get_downloads(test_session)) == 1
|
|
|
|
# Remove the frequency.
|
|
new_channel = dict(
|
|
directory=channel['directory'],
|
|
name=channel['name'],
|
|
url=channel['url'],
|
|
download_frequency=None,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
def test_channel_frequency_update(download_channel, test_client, test_session, test_download_manager):
|
|
"""
|
|
A Channel's Download record is updated when the Channel's frequency is updated.
|
|
"""
|
|
old_frequency = download_channel.get_download().frequency
|
|
assert old_frequency
|
|
|
|
data = dict(
|
|
directory=str(download_channel.directory),
|
|
name=download_channel.name,
|
|
url=download_channel.url,
|
|
download_frequency=100,
|
|
)
|
|
request, response = test_client.put(f'/api/videos/channels/{download_channel.id}', json=data)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
download = download_channel.get_download()
|
|
assert download.frequency == 100
|
|
|
|
# Only one download
|
|
downloads = test_session.query(Download).all()
|
|
assert len(list(downloads)) == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_channel_download_crud(test_session, simple_channel, test_downloader_config):
|
|
"""Modifying a Channel modifies it's Download."""
|
|
assert simple_channel.url
|
|
assert simple_channel.download_frequency is None
|
|
|
|
simple_channel.update({'download_frequency': 1000})
|
|
test_session.commit()
|
|
|
|
download: Download = test_session.query(Download).one()
|
|
assert simple_channel.get_download() == download
|
|
assert simple_channel.url == download.url
|
|
assert simple_channel.download_frequency == download.frequency
|
|
|
|
# Increase the frequency to 1 second.
|
|
simple_channel.update({'download_frequency': 1})
|
|
test_session.commit()
|
|
assert simple_channel.url == download.url
|
|
assert simple_channel.download_frequency == download.frequency == 1
|
|
assert test_session.query(Download).count() == 1
|
|
|
|
# Changing the Channel's URL changes the download's URL.
|
|
simple_channel.update({'url': 'https://example.com/new url'})
|
|
test_session.commit()
|
|
assert simple_channel.url == 'https://example.com/new url'
|
|
assert simple_channel.download_frequency == download.frequency == 1
|
|
assert test_session.query(Download).count() == 1
|
|
|
|
# Removing the Channel's frequency removes the download.
|
|
simple_channel.update({'download_frequency': None})
|
|
test_session.commit()
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
# Removing the URL is supported.
|
|
simple_channel.update({'url': None})
|
|
test_session.commit()
|
|
|
|
# Adding a URL is supported.
|
|
simple_channel.update({'url': 'https://example.com'})
|
|
test_session.commit()
|
|
|
|
# Adding a frequency adds a Download.
|
|
simple_channel.update({'download_frequency': 1})
|
|
test_session.commit()
|
|
assert test_session.query(Download).count() == 1
|
|
next_download = simple_channel.get_download().next_download
|
|
assert next_download
|
|
|
|
# Next download isn't overwritten.
|
|
time.sleep(1)
|
|
simple_channel.update({'name': 'new name'})
|
|
test_session.commit()
|
|
assert simple_channel.get_download().next_download == next_download
|
|
|
|
# Deleting a Channel deletes it's Download.
|
|
delete_channel(channel_id=simple_channel.id)
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
|
|
def test_video_file_name(test_session, simple_video, test_client):
|
|
"""If a Video has no title, the front-end can use the file name as the title."""
|
|
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
|
|
assert resp.status_code == HTTPStatus.OK
|
|
assert resp.json['file_group']['video']['video_path'] == 'simple_video.mp4'
|
|
assert resp.json['file_group']['video'].get('stem') == 'simple_video'
|
|
|
|
|
|
def test_channel_conflicts(test_client, test_session, test_directory):
|
|
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory,
|
|
match_regex='asdf',
|
|
name='Example Channel 1',
|
|
url='https://example.com/channel1',
|
|
)
|
|
|
|
def _post_channel(channel):
|
|
return test_client.post('/api/videos/channels', content=json.dumps(channel))
|
|
|
|
# Create it
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
# Name is an exact match
|
|
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory2).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory2,
|
|
name='Example Channel 1',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'error': 'The channel name is already taken.', 'code': 5}}
|
|
|
|
# Directory was already used
|
|
new_channel = dict(
|
|
directory=channel_directory,
|
|
name='name is fine',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'code': 7, 'error': 'The directory is already used by another channel.'}}
|
|
|
|
# URL is already used
|
|
channel_directory3 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory3).mkdir()
|
|
new_channel = dict(
|
|
directory=channel_directory3,
|
|
name='name is fine',
|
|
url='https://example.com/channel1',
|
|
)
|
|
request, response = _post_channel(new_channel)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
|
'cause': {'code': 6, 'error': 'The URL is already used by another channel.'}}
|
|
|
|
|
|
def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory):
|
|
"""Two channels with empty URLs shouldn't conflict"""
|
|
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory).mkdir()
|
|
|
|
new_channel = {
|
|
'name': 'Fooz',
|
|
'directory': channel_directory,
|
|
}
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
location = response.headers['Location']
|
|
|
|
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
|
pathlib.Path(channel_directory2).mkdir()
|
|
new_channel = {
|
|
'name': 'Barz',
|
|
'directory': channel_directory2,
|
|
}
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
assert location != response.headers['Location']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_channel_download_requires_refresh(test_session, download_channel, video_download_manager, video_factory,
|
|
events_history):
|
|
"""A Channel cannot be downloaded until it has been refreshed.
|
|
|
|
Videos already downloaded are not downloaded again."""
|
|
vid = video_factory(channel_id=download_channel.id, with_video_file=True, with_poster_ext='jpg')
|
|
vid.url = 'https://example.com/1'
|
|
d = download_channel.get_download()
|
|
test_session.commit()
|
|
assert not d.next_download
|
|
|
|
def assert_refreshed(expected: bool):
|
|
channel: Channel = test_session.query(Channel).one()
|
|
assert channel.refreshed == expected
|
|
if expected:
|
|
assert channel.info_json
|
|
else:
|
|
assert not channel.info_json
|
|
|
|
assert_refreshed(False)
|
|
test_session.commit()
|
|
|
|
downloaded_urls = []
|
|
|
|
async def do_download(_, download):
|
|
# Track all downloaded URLs.
|
|
downloaded_urls.append(download.url)
|
|
return DownloadResult(success=True)
|
|
|
|
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
|
mock.patch('modules.videos.downloader.VideoDownloader.do_download', do_download), \
|
|
mock.patch('wrolpi.files.lib.apply_indexers'): # TODO why does apply_indexers break this test?
|
|
entries = [
|
|
dict(id=vid.source_id, view_count=0, webpage_url='https://example.com/1'), # Already downloaded.
|
|
dict(id='not downloaded', view_count=0, webpage_url='https://example.com/2'),
|
|
]
|
|
mock_extract_info.return_value = {'entries': entries, 'url': 'foo', 'uploader': 'some uploader',
|
|
'channel_id': 'the id', 'id': 'the id'}
|
|
await video_download_manager.do_downloads()
|
|
await video_download_manager.wait_for_all_downloads()
|
|
|
|
# Channel was refreshed before downloading videos.
|
|
assert_refreshed(True)
|
|
assert d.next_download
|
|
# Only the missing video was downloaded.
|
|
assert downloaded_urls == ['https://example.com/2']
|
|
|
|
# Should not send refresh events because downloads are automated.
|
|
assert events_history == []
|
|
|
|
|
|
def test_channel_post_directory(test_session, test_client, test_directory):
|
|
"""A Channel can be created with or without an existing directory."""
|
|
# Channel can be created with a directory which is not on disk.
|
|
data = dict(name='foo', directory='foo')
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
directory = test_session.query(Channel).filter_by(id=1).one().directory
|
|
assert (test_directory / 'foo') == directory
|
|
assert not directory.is_dir()
|
|
assert directory.is_absolute()
|
|
|
|
# Channel can be created and have its directory be created.
|
|
data = dict(name='bar', directory='bar', mkdir=True)
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
directory = test_session.query(Channel).filter_by(id=2).one().directory
|
|
assert (test_directory / 'bar') == directory
|
|
assert directory.is_dir()
|
|
assert directory.is_absolute()
|
|
|
|
|
|
def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
|
|
request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}')
|
|
assert response.status_code == HTTPStatus.OK
|
|
|
|
|
|
def test_channel_crud(test_session, test_client, test_directory, test_download_manager):
|
|
channel_directory = test_directory / 'channel directory'
|
|
channel_directory.mkdir()
|
|
|
|
new_channel = dict(
|
|
directory=str(channel_directory),
|
|
match_regex='asdf',
|
|
name=' Example Channel 1 ',
|
|
url='https://example.com/channel1',
|
|
)
|
|
|
|
# Channel doesn't exist
|
|
request, response = test_client.get('/api/videos/channels/examplechannel1')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
|
|
|
|
# Create it
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
location = response.headers['Location']
|
|
|
|
request, response = test_client.get(location)
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
created = response.json['channel']
|
|
assert created
|
|
assert created['id']
|
|
# No frequency was provided. No download exists.
|
|
assert not created['download_frequency']
|
|
assert test_session.query(Download).count() == 0
|
|
|
|
# Channel name leading/trailing whitespace should be stripped
|
|
assert created['name'] == 'Example Channel 1'
|
|
|
|
# Channel directory should be relative to the media directory
|
|
assert not pathlib.Path(created['directory']).is_absolute(), \
|
|
f'Channel directory is absolute: {created["directory"]}'
|
|
|
|
# Can't create it again
|
|
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
def put_and_fetch(new_channel_):
|
|
request, response = test_client.put(location, content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
|
request, response = test_client.get(location)
|
|
assert response.status_code == HTTPStatus.OK
|
|
channel = response.json['channel']
|
|
return channel
|
|
|
|
# Update it
|
|
new_channel['name'] = 'Example Channel 2'
|
|
new_channel['directory'] = str(new_channel['directory']) # noqa
|
|
new_channel['download_frequency'] = 60
|
|
channel = put_and_fetch(new_channel)
|
|
assert channel['id'] == 1
|
|
assert channel['name'] == 'Example Channel 2'
|
|
assert channel['directory'] == channel_directory.name
|
|
assert channel['match_regex'] == 'asdf'
|
|
assert channel['url'] == 'https://example.com/channel1'
|
|
# Download was created.
|
|
assert channel['download_frequency']
|
|
assert (downloads := test_download_manager.get_downloads(test_session)) and len(downloads) == 1 and downloads[
|
|
0].url == new_channel['url']
|
|
|
|
# Update with no download frequency.
|
|
new_channel['download_frequency'] = None
|
|
channel = put_and_fetch(new_channel)
|
|
assert channel['download_frequency'] is None
|
|
# Download was deleted.
|
|
assert len(test_download_manager.get_downloads(test_session)) == 0
|
|
|
|
# Can't update channel that doesn't exist
|
|
request, response = test_client.put('/api/videos/channels/doesnt_exist',
|
|
content=json.dumps(new_channel))
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
# Delete the new channel
|
|
request, response = test_client.delete(location)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Cant delete it again
|
|
request, response = test_client.delete(location)
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
|
|
def test_change_channel_url(test_client, test_session, test_download_manager, download_channel,
|
|
test_download_manager_config):
|
|
# Change the Channel's URL.
|
|
download_channel.update({'url': 'https://example.com/new-url'})
|
|
test_session.commit()
|
|
|
|
download = test_session.query(Download).one()
|
|
assert download.url == download_channel.url == 'https://example.com/new-url', "Channel's download URL was not changed."
|
|
|
|
download_channel.update({'download_frequency': None})
|
|
assert not list(test_session.query(Download).all())
|
|
|
|
|
|
def test_search_videos_channel(test_client, test_session, video_factory):
|
|
with get_db_session(commit=True) as session:
|
|
channel1 = Channel(name='Foo')
|
|
channel2 = Channel(name='Bar')
|
|
session.add(channel1)
|
|
session.add(channel2)
|
|
session.flush()
|
|
session.refresh(channel1)
|
|
session.refresh(channel2)
|
|
|
|
# Channels don't have videos yet
|
|
d = dict(channel_id=channel1.id)
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['file_groups']) == 0
|
|
|
|
with get_db_session(commit=True) as session:
|
|
vid1 = video_factory(title='vid1', channel_id=channel2.id)
|
|
vid2 = video_factory(title='vid2', channel_id=channel1.id)
|
|
session.add(vid1)
|
|
session.add(vid2)
|
|
|
|
# Videos are gotten by their respective channels
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['file_groups']) == 1
|
|
assert response.json['totals']['file_groups'] == 1
|
|
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid2.mp4', video=dict(channel_id=channel1.id)))
|
|
|
|
d = dict(channel_id=channel2.id)
|
|
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert len(response.json['file_groups']) == 1
|
|
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
|
|
|
|
|
|
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory, assert_video_search):
|
|
for i in range(50):
|
|
video_factory(channel_id=simple_channel.id)
|
|
|
|
channel2 = Channel(name='Bar')
|
|
test_session.add(channel2)
|
|
test_session.flush()
|
|
test_session.refresh(channel2)
|
|
video_factory(channel_id=channel2.id)
|
|
video_factory(title='vid2', channel_id=channel2.id, with_video_file=True)
|
|
test_session.commit()
|
|
|
|
# Get first, second, third, and empty pages of videos.
|
|
tests = [
|
|
# (offset, video_count)
|
|
(0, 20),
|
|
(20, 20),
|
|
(40, 10),
|
|
(50, 0),
|
|
]
|
|
last_ids = []
|
|
for offset, video_count in tests:
|
|
_, response = assert_video_search(channel_id=simple_channel.id, order_by='id', offset=offset, limit=20)
|
|
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
|
|
current_ids = [i['id'] for i in response.json['file_groups']]
|
|
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
|
last_ids = current_ids
|