mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 02:48:38 -04:00
8008cd1fd8
COUNT(*) OVER() forced SQLite to visit every matching file_group before LIMIT, which is a full table scan of the (large) file_group table — ~22s cold on a Pi for videos ordered by download date. Count separately and cache for 60s by filters so indexed orders can stop after one page. Past-the-end pages now report the true total. Videos/archives/docs pagers keep totalPages across page changes.
762 lines
37 KiB
Python
762 lines
37 KiB
Python
import asyncio
|
|
import threading
|
|
from unittest import mock
|
|
import json
|
|
from http import HTTPStatus
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from modules.videos.models import Video, Channel
|
|
from modules.videos.video import lib as video_lib
|
|
from wrolpi.common import get_wrolpi_config
|
|
from wrolpi.db import get_db_curs
|
|
from wrolpi.downloader import Download
|
|
from wrolpi.files.models import FileGroup
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_refresh_videos_index(async_client, test_session, test_directory, video_factory, refresh_files):
|
|
"""The video modeler indexes video data into the Video's FileGroup."""
|
|
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
|
|
test_session.commit()
|
|
|
|
test_session.query(FileGroup).delete()
|
|
test_session.commit()
|
|
|
|
for _ in range(1):
|
|
# Refreshing twice does not change the results.
|
|
await refresh_files()
|
|
|
|
assert test_session.query(FileGroup).count() == 1
|
|
video: Video = test_session.query(Video).one()
|
|
assert video.file_group.indexed is True, 'Video was not indexed'
|
|
assert video.file_group.a_text and video.file_group.title, 'Video title was not indexed'
|
|
assert video.file_group.d_text, 'Video captions were not indexed'
|
|
|
|
# Delete caption file, the video should be indexed again.
|
|
video: Video = test_session.query(Video).one()
|
|
for path in video.caption_paths:
|
|
path.unlink()
|
|
|
|
await refresh_files()
|
|
assert video.file_group.indexed is True, 'Video was not indexed'
|
|
assert video.file_group.a_text and video.file_group.title, 'Video title was not indexed'
|
|
assert not video.file_group.d_text, 'Video captions were not removed'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_refresh_videos(async_client, test_session, test_directory, simple_channel, video_factory,
|
|
video_file_factory, image_bytes_factory):
|
|
subdir = test_directory / 'subdir'
|
|
subdir.mkdir()
|
|
|
|
# video1 is in a subdirectory, move its files into the subdirectory.
|
|
video1 = video_factory(channel_id=simple_channel.id, with_video_file=subdir / 'video1.mp4',
|
|
with_info_json=True, with_poster_ext='jpg')
|
|
test_session.commit()
|
|
video1.file_group.size = video1.file_group.data = None
|
|
video1.file_group.files = []
|
|
# video2 is in the test directory.
|
|
video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
|
|
with_poster_ext='jpg')
|
|
video2.file_group.data = None
|
|
video2.file_group.files = []
|
|
test_session.commit()
|
|
|
|
assert not video1.file_group.size, 'video1 should not have size during creation'
|
|
|
|
# Create a video not in the DB.
|
|
vid3 = video_file_factory(test_directory / 'vid3.mp4')
|
|
|
|
# Orphan poster should be ignored.
|
|
orphan_poster = Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg')
|
|
orphan_poster.write_bytes(image_bytes_factory())
|
|
|
|
# Create a bogus video in the channel (file doesn't exist, should be removed during refresh).
|
|
video_file = test_directory / 'foo.mp4'
|
|
with get_db_curs(commit=True) as curs:
|
|
stmt = "INSERT INTO file_group (mimetype, directory, primary_path, indexed, files, model)" \
|
|
" values ('video/mp4', :directory, :primary_path, true, :files, 'video') RETURNING id"
|
|
params = {'directory': str(test_directory), 'primary_path': str(video_file), 'files': '[]'}
|
|
curs.execute(stmt, params)
|
|
video4_id = curs.fetchall()[0][0]
|
|
stmt = "INSERT INTO video (file_group_id) values (:video_id)"
|
|
curs.execute(stmt, {'video_id': str(video4_id)})
|
|
|
|
await async_client.post('/api/files/refresh')
|
|
|
|
# Posters were found during refresh.
|
|
assert video1.file_group.my_poster_files()
|
|
assert 'subdir' in str(video1.file_group.my_poster_files()[0])
|
|
assert video2.file_group.my_poster_files()
|
|
# Missing video3 was found
|
|
video3: Video = test_session.query(Video).filter_by(id=4).one()
|
|
assert video3.video_path == vid3
|
|
assert len(set(test_session.query(FileGroup.primary_path).filter(FileGroup.model == 'video'))) == 3, \
|
|
'Should have 3 distinct video files'
|
|
# Bogus video was removed.
|
|
assert not any('foo.mp4' in str(i.video_path) for i in test_session.query(Video).all())
|
|
# Orphan file was not deleted.
|
|
assert orphan_poster.is_file(), 'Orphan poster was removed!'
|
|
|
|
assert video1.file_group.size, 'video1 size was not found'
|
|
|
|
# Remove video1's poster, video1 should be updated.
|
|
video1_video_path = str(video1.video_path)
|
|
video1.poster_path.unlink()
|
|
await async_client.post('/api/files/refresh')
|
|
video1 = Video.get_by_path(test_session, video1_video_path)
|
|
assert not video1.poster_path
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_channels_with_videos(test_session, async_client, test_directory, channel_factory, video_factory):
|
|
channel1 = channel_factory('channel1', name='channel1')
|
|
channel2 = channel_factory('channel2', name='channel2')
|
|
vid1 = video_factory(channel_id=channel1.id, with_video_file=True)
|
|
vid2 = video_factory(channel_id=channel2.id, with_video_file=True)
|
|
vid3 = video_factory(with_video_file=True)
|
|
test_session.commit()
|
|
|
|
vid1_path = vid1.video_path
|
|
vid2_path = vid2.video_path
|
|
vid3_path = vid3.video_path
|
|
|
|
test_session.delete(vid1)
|
|
test_session.delete(vid2)
|
|
test_session.delete(vid3)
|
|
test_session.commit()
|
|
|
|
assert test_session.query(Video).count() == 0, 'Videos were not deleted'
|
|
assert test_session.query(Channel).count() == 2, 'Channels were deleted!'
|
|
assert vid1_path.is_file(), 'Video file was deleted'
|
|
assert vid2_path.is_file(), 'Video file was deleted'
|
|
assert vid3_path.is_file(), 'Video file was deleted'
|
|
|
|
await async_client.post('/api/files/refresh')
|
|
|
|
assert test_session.query(Video).count() == 3, 'Did not find correct amount of video files.'
|
|
from wrolpi.collections import Collection
|
|
assert {i[0] for i in test_session.query(Collection.name).join(Channel).filter(Collection.kind == 'channel')} == {
|
|
'channel1', 'channel2'}, 'Channels were changed.'
|
|
|
|
vid1 = Video.get_by_path(test_session, vid1_path)
|
|
vid2 = Video.get_by_path(test_session, vid2_path)
|
|
vid3 = Video.get_by_path(test_session, vid3_path)
|
|
|
|
assert vid1.channel == channel1
|
|
assert vid2.channel == channel2
|
|
assert vid3.channel is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_api_download(test_session, async_client, test_directory):
|
|
"""A video can be downloaded."""
|
|
content = dict(urls=['https://example.com/video1', ], downloader='video', destination='dest',
|
|
settings=dict(excluded_urls='example.com'))
|
|
request, response = await async_client.post('/api/download', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
download = test_session.query(Download).one()
|
|
assert download.url == 'https://example.com/video1'
|
|
assert download.downloader == 'video'
|
|
assert download.settings['excluded_urls'] == 'example.com'
|
|
assert download.destination == test_directory / 'dest'
|
|
assert not download.settings.get('tag_names')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_api_download_video_tags(test_session, async_client, tag_factory):
|
|
"""A user can request Tags for a video being downloaded."""
|
|
tag1 = await tag_factory()
|
|
tag2 = await tag_factory()
|
|
|
|
content = dict(urls=['https://example.com/video1', ], downloader='video', tag_names=[tag1.name, tag2.name])
|
|
request, response = await async_client.post('/api/download', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
download = test_session.query(Download).one()
|
|
assert download.url == 'https://example.com/video1'
|
|
assert download.downloader == 'video'
|
|
assert download.tag_names == [tag1.name, tag2.name]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_videos_file(test_session, test_directory, video_with_search_factory,
|
|
assert_video_search):
|
|
"""Test that videos can be searched and that their order is by their textsearch rank."""
|
|
# These captions have repeated letters, so they will be higher in the ranking.
|
|
videos = [
|
|
('1', 'b b b b e d d'),
|
|
('2', '2 b b b d'),
|
|
('3', 'b b'),
|
|
('4', 'b e e'),
|
|
('5', ''),
|
|
]
|
|
for title, caption in videos:
|
|
video_with_search_factory(title=title, d_text=caption)
|
|
test_session.commit()
|
|
|
|
# Captions are d_text, so a deep search is required to match them.
|
|
# Repeated runs should return the same result
|
|
for _ in range(2):
|
|
# Only videos with a b are returned, ordered by their rank. (FTS5's bm25 normalizes by
|
|
# document length, so the short 'b b' caption outranks longer captions with more b's;
|
|
# Postgres' ts_rank ordered by raw frequency.)
|
|
await assert_video_search(search_str='b', deep=True, assert_ids=[3, 1, 2, 4])
|
|
|
|
# The default (fast) search does not match captions.
|
|
await assert_video_search(search_str='b', assert_ids=[])
|
|
|
|
# Only two captions have e
|
|
await assert_video_search(search_str='e', deep=True, assert_ids=[4, 1])
|
|
|
|
# Only 2 contains 2
|
|
await assert_video_search(search_str='2', deep=True, assert_ids=[2, ])
|
|
|
|
# Only two captions have d
|
|
await assert_video_search(search_str='d', deep=True, assert_ids=[1, 2])
|
|
|
|
# 5 can be gotten by it's title, in either mode.
|
|
await assert_video_search(search_str='5', assert_ids=[5, ])
|
|
await assert_video_search(search_str='5', deep=True, assert_ids=[5, ])
|
|
|
|
# only video 1 has e and d
|
|
await assert_video_search(search_str='e d', deep=True, assert_ids=[1, ])
|
|
|
|
# Videos 1 and 4 have b and e; 4's caption is shorter so bm25 ranks it higher.
|
|
await assert_video_search(search_str='b e', deep=True, assert_ids=[4, 1])
|
|
|
|
# Check totals are correct even with a limit
|
|
await assert_video_search(search_str='b', deep=True, limit=2, assert_ids=[3, 1], assert_total=4)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory):
|
|
"""Search the Video table. This does not need to use a join with the File table."""
|
|
vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
|
|
vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2',
|
|
channel_id=simple_channel.id)
|
|
vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
|
|
tag1, tag2 = await tag_factory(), await tag_factory()
|
|
|
|
vid1.add_tag(test_session, tag1.id)
|
|
|
|
# vid2 has two tags
|
|
vid2.add_tag(test_session, tag1.id)
|
|
vid2.add_tag(test_session, tag2.id)
|
|
|
|
vid3.add_tag(test_session, tag2.id)
|
|
|
|
test_session.commit()
|
|
assert test_session.query(Video).count() == 3
|
|
assert test_session.query(FileGroup).count() == 3
|
|
|
|
await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime')
|
|
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime')
|
|
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2)
|
|
await assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2,
|
|
offset=1)
|
|
await assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime',
|
|
channel_id=simple_channel.id)
|
|
# No search_str, so the default 'rank' order is a constant; ties fall to fg.id DESC.
|
|
await assert_video_search(assert_total=2, assert_ids=[vid2.id, vid1.id], tag_names=[tag1.name])
|
|
await assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
|
|
await assert_video_search(assert_total=2, assert_ids=[vid2.id], tag_names=[tag1.name], limit=1)
|
|
# Only vid2 has both tags.
|
|
await assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name])
|
|
|
|
# No results, no total is returned from the DB.
|
|
await assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2)
|
|
|
|
# Check all order_by.
|
|
for order_by in video_lib.VIDEO_ORDERS:
|
|
await assert_video_search(order_by=order_by)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_videos_censored(test_session, video_factory, assert_video_search):
|
|
"""The censored filter returns only Videos which are no longer available for download."""
|
|
vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
|
|
vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2')
|
|
vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
|
|
|
|
# vid2 is no longer available.
|
|
vid2.file_group.censored = True
|
|
test_session.commit()
|
|
|
|
# Without the filter, all Videos are returned.
|
|
await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id],
|
|
order_by='published_datetime')
|
|
await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id],
|
|
order_by='published_datetime', censored=False)
|
|
|
|
# With the filter, only the censored Video is returned.
|
|
await assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='published_datetime', censored=True)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_audio_only(test_session, audio_factory, assert_video_search):
|
|
"""Audio-only files should be searchable alongside videos."""
|
|
audio1 = audio_factory(title='podcast episode one', upload_date='2025-01-01', with_info_json=True)
|
|
audio2 = audio_factory(title='podcast episode two', upload_date='2025-01-02', with_info_json=True)
|
|
test_session.commit()
|
|
|
|
assert audio1.file_group.mimetype == 'audio/mpeg'
|
|
assert audio1.file_group.model == 'video'
|
|
|
|
# Both audio files are returned.
|
|
await assert_video_search(assert_total=2, order_by='published_datetime')
|
|
|
|
# Search by title.
|
|
await assert_video_search(search_str='podcast', assert_total=2)
|
|
await assert_video_search(search_str='episode one', assert_total=1, assert_ids=[audio1.id])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_mixed_video_and_audio(test_session, video_factory, audio_factory, assert_video_search):
|
|
"""Audio and video files should both appear in search results."""
|
|
video = video_factory(title='tutorial video', upload_date='2025-01-01', with_video_file=True)
|
|
audio = audio_factory(title='tutorial audio', upload_date='2025-01-02', with_info_json=True)
|
|
test_session.commit()
|
|
|
|
# Both appear when searching.
|
|
await assert_video_search(assert_total=2, order_by='published_datetime')
|
|
await assert_video_search(search_str='tutorial', assert_total=2)
|
|
|
|
# Each can be found individually.
|
|
await assert_video_search(search_str='video', assert_total=1, assert_ids=[video.id])
|
|
await assert_video_search(search_str='audio', assert_total=1, assert_ids=[audio.id])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_video(async_client, test_session, video_factory, test_download_manager):
|
|
"""When a Video record is deleted, the FileGroup should be deleted. The URL is added to the
|
|
global skip list so the video is not downloaded again."""
|
|
video = video_factory(with_video_file=True, with_poster_ext='png')
|
|
video.file_group.url = 'https://example.com/video/1'
|
|
files = video.file_group.my_paths()
|
|
test_session.commit()
|
|
|
|
body = json.dumps({'file_group_ids': [video.file_group_id]})
|
|
request, response = await async_client.post('/api/files/delete_groups', content=body)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
assert test_session.query(Video).count() == 0, 'Video was not deleted.'
|
|
assert test_session.query(FileGroup).count() == 0, 'Video FileGroup was not deleted.'
|
|
assert not any(i.exists() for i in files), 'All files should have been deleted.'
|
|
assert test_download_manager.is_skipped('https://example.com/video/1'), 'Video should not be downloaded again.'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_format_videos_description(async_client, test_session, test_directory, channel_factory, tag_factory):
|
|
channel = channel_factory(name='Channel Name')
|
|
tag = await tag_factory()
|
|
test_session.commit()
|
|
|
|
body = dict()
|
|
request, response = await async_client.post('/api/videos/tag_info', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['videos_destination'] == 'videos'
|
|
|
|
body = dict(channel_id=channel.id)
|
|
request, response = await async_client.post('/api/videos/tag_info', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['videos_destination'] == 'videos/Channel Name'
|
|
|
|
body = dict(channel_id=channel.id, tag_name=tag.name)
|
|
request, response = await async_client.post('/api/videos/tag_info', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['videos_destination'] == 'videos/one/Channel Name'
|
|
|
|
get_wrolpi_config().videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s'
|
|
body = dict(channel_id=channel.id, tag_name=tag.name)
|
|
request, response = await async_client.post('/api/videos/tag_info', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_size_to_duration_sort(test_session, video_factory, assert_video_search):
|
|
"""Test the 'size_to_duration' sort option."""
|
|
# Create videos with different size-to-duration ratios
|
|
# All videos will have the same size (from the test file) but different durations
|
|
|
|
# Video with 10 second duration (highest ratio)
|
|
vid1 = video_factory(with_video_file=True, title='vid1', with_info_json={'duration': 10})
|
|
|
|
# Video with 20 second duration (medium ratio)
|
|
vid2 = video_factory(with_video_file=True, title='vid2', with_info_json={'duration': 20})
|
|
|
|
# Video with 30 second duration (lowest ratio)
|
|
vid3 = video_factory(with_video_file=True, title='vid3', with_info_json={'duration': 30})
|
|
|
|
test_session.commit()
|
|
|
|
# Verify that the videos have the expected size and length values
|
|
vid1 = test_session.query(Video).filter_by(id=vid1.id).one()
|
|
vid2 = test_session.query(Video).filter_by(id=vid2.id).one()
|
|
vid3 = test_session.query(Video).filter_by(id=vid3.id).one()
|
|
|
|
# All videos should have the same size
|
|
assert vid1.file_group.size == vid2.file_group.size == vid3.file_group.size
|
|
|
|
# But different lengths
|
|
assert vid1.file_group.length == 10
|
|
assert vid2.file_group.length == 20
|
|
assert vid3.file_group.length == 30
|
|
|
|
# Test ascending order (lowest ratio first)
|
|
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='size_to_duration')
|
|
|
|
# Test descending order (highest ratio first)
|
|
await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='-size_to_duration')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_file_format(async_client, test_session, fake_now):
|
|
body = dict(video_file_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s')
|
|
request, response = await async_client.post('/api/videos/file_format', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['preview'] == 'WROLPi_20000101_Qz-FuenRylQ_The title of the video.mp4'
|
|
|
|
body = dict(video_file_format='%(upload_date)s_%(id)s_%(title)s.%(ext)s')
|
|
request, response = await async_client.post('/api/videos/file_format', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['preview'] == '20000101_Qz-FuenRylQ_The title of the video.mp4'
|
|
|
|
body = dict(video_file_format='%(upload_date)s_%(id)s_%(title)s')
|
|
request, response = await async_client.post('/api/videos/file_format', json=body)
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json['error'] == 'Filename must end with .%(ext)s'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_file_format_with_upload_year(async_client, test_session, fake_now):
|
|
"""Video file format API handles upload_year and upload_month variables."""
|
|
body = dict(video_file_format='%(upload_year)s/%(upload_month)s/%(title)s.%(ext)s')
|
|
request, response = await async_client.post('/api/videos/file_format', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['preview'] == '2000/01/The title of the video.mp4'
|
|
|
|
body = dict(video_file_format='%(upload_year)s/%(uploader)s_%(upload_date)s_%(title)s.%(ext)s')
|
|
request, response = await async_client.post('/api/videos/file_format', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['preview'] == '2000/WROLPi_20000101_The title of the video.mp4'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_upload_file_tracking(test_session, async_client, video_factory, make_multipart_form,
|
|
image_bytes_factory):
|
|
"""Test that uploading info.json, poster, and caption files via /api/files/upload properly tracks them in Video and FileGroup."""
|
|
from wrolpi.common import get_relative_to_media_directory
|
|
|
|
# Step 1: Create a Video with just a video file (no info.json, poster, or captions)
|
|
video = video_factory(with_video_file=True, title='file_tracking_test')
|
|
test_session.commit()
|
|
|
|
# Step 2: Verify Video and FileGroup are created, but optional files do not exist
|
|
file_group = video.file_group
|
|
assert not video.info_json_path, 'info.json should not exist yet.'
|
|
assert not video.poster_path, 'Poster should not exist yet.'
|
|
assert not video.caption_paths, 'Captions should not exist yet.'
|
|
assert file_group.files is not None, 'FileGroup should have files array'
|
|
initial_file_count = len(file_group.files)
|
|
|
|
# Step 3: Upload info.json via /api/files/upload
|
|
# Extract the video stem (e.g., "file_tracking_test")
|
|
video_stem = video.video_path.stem
|
|
|
|
# Create info.json content
|
|
info_json_data = {
|
|
'title': 'file_tracking_test',
|
|
'duration': 60,
|
|
'uploader': 'test_uploader'
|
|
}
|
|
info_json_bytes = json.dumps(info_json_data).encode()
|
|
|
|
# Get relative destination directory
|
|
destination = str(get_relative_to_media_directory(video.video_path.parent))
|
|
filename = f'{video_stem}.info.json'
|
|
|
|
# POST to /api/files/upload
|
|
forms = [
|
|
dict(name='destination', value=destination),
|
|
dict(name='filename', value=filename),
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='totalChunks', value='0'),
|
|
dict(name='chunkSize', value=str(len(info_json_bytes))),
|
|
dict(name='overwrite', value='false'),
|
|
dict(name='chunk', value=info_json_bytes, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
|
|
|
|
# Step 4: Verify info.json is tracked
|
|
video = test_session.query(Video).filter_by(id=video.id).one()
|
|
|
|
# Assert info.json properties exist
|
|
assert video.info_json_path is not None, 'Video.info_json_path should exist'
|
|
assert video.info_json_file is not None, 'Video.info_json_file should exist'
|
|
|
|
# Assert FileGroup has the correct number of files
|
|
assert len(video.file_group.files) == initial_file_count + 1, \
|
|
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
|
|
assert video.file_group.data.get('info_json_path'), \
|
|
'Video info.json should be in FileGroup.data'
|
|
|
|
# Step 5: Upload poster image and verify it's added to the FileGroup
|
|
file_count_before_poster = len(video.file_group.files)
|
|
|
|
# Create image using factory
|
|
image_data = image_bytes_factory()
|
|
|
|
# Upload image with same stem as the Video (using .jpg)
|
|
poster_filename = f'{video_stem}.jpg'
|
|
forms = [
|
|
dict(name='destination', value=destination),
|
|
dict(name='filename', value=poster_filename),
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='totalChunks', value='0'),
|
|
dict(name='chunkSize', value=str(len(image_data))),
|
|
dict(name='overwrite', value='false'),
|
|
dict(name='chunk', value=image_data, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.CREATED, f'Poster upload failed with status {response.status_code}: {response.json}'
|
|
|
|
video = test_session.query(Video).filter_by(id=video.id).one()
|
|
|
|
# Assert poster properties exist
|
|
assert video.poster_path is not None, 'Video.poster_path should exist'
|
|
assert video.poster_file is not None, 'Video.poster_file should exist'
|
|
# data['poster_path'] stores just the filename, poster_path resolves to absolute
|
|
assert video.file_group.data.get('poster_path') == video.poster_path.name, 'Video poster should be in FileGroup.data'
|
|
|
|
# Assert poster is in FileGroup.files (use my_files() to get resolved absolute paths)
|
|
file_paths = [f['path'] for f in video.file_group.my_files()]
|
|
assert video.poster_path in file_paths, 'Poster should be in FileGroup.files'
|
|
|
|
# Assert FileGroup has correct number of files
|
|
assert len(video.file_group.files) == file_count_before_poster + 1, \
|
|
f'FileGroup should have {file_count_before_poster + 1} files after adding poster'
|
|
|
|
# Step 6: Upload caption file and verify it's added to the FileGroup
|
|
file_count_before_caption = len(video.file_group.files)
|
|
|
|
# Create caption content
|
|
caption_content = """WEBVTT
|
|
|
|
00:00:00.000 --> 00:00:02.000
|
|
This is a test caption.
|
|
|
|
00:00:02.000 --> 00:00:04.000
|
|
Another line of captions.
|
|
"""
|
|
|
|
# Upload caption with same stem as the Video
|
|
caption_filename = f'{video_stem}.en.vtt'
|
|
caption_bytes = caption_content.encode()
|
|
forms = [
|
|
dict(name='destination', value=destination),
|
|
dict(name='filename', value=caption_filename),
|
|
dict(name='chunkNumber', value='0'),
|
|
dict(name='totalChunks', value='0'),
|
|
dict(name='chunkSize', value=str(len(caption_bytes))),
|
|
dict(name='overwrite', value='false'),
|
|
dict(name='chunk', value=caption_bytes, filename='chunk'),
|
|
]
|
|
body = make_multipart_form(forms)
|
|
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
|
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
|
assert response.status_code == HTTPStatus.CREATED, f'Caption upload failed with status {response.status_code}: {response.json}'
|
|
|
|
video = test_session.query(Video).filter_by(id=video.id).one()
|
|
|
|
# Assert caption properties exist
|
|
assert video.caption_paths, 'Video.caption_paths should exist'
|
|
assert len(video.caption_paths) == 1, 'Should have exactly one caption file'
|
|
assert video.caption_files, 'Video.caption_files should exist'
|
|
assert video.file_group.data.get('caption_paths'), 'Video captions should be in FileGroup.data'
|
|
|
|
# Assert caption is in FileGroup.files (use my_files() to get resolved absolute paths)
|
|
file_paths = [f['path'] for f in video.file_group.my_files()]
|
|
assert video.caption_paths[0] in file_paths, 'Caption should be in FileGroup.files'
|
|
|
|
# Assert FileGroup has correct number of files
|
|
assert len(video.file_group.files) == file_count_before_caption + 1, \
|
|
f'FileGroup should have {file_count_before_caption + 1} files after adding caption'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_videos_does_not_block_event_loop(async_client):
|
|
"""A slow video search must not stall other requests on the same Sanic worker.
|
|
|
|
The search runs a synchronous SQLite query; on a large library it takes seconds cold. If it
|
|
runs on the event loop, every other request on the worker waits for it.
|
|
|
|
The search is held open with an Event and the test asserts the echo completes while the search
|
|
is still inside the query, rather than timing the echo against a wall-clock threshold, so the
|
|
test cannot flake on a loaded CI machine. The events are only ever waited on off the event
|
|
loop, so a regression (query on the loop) cannot deadlock the test; it fails the ordering
|
|
assertion instead."""
|
|
await async_client.get('/api/echo')
|
|
|
|
entered = threading.Event() # the handler has reached the search query.
|
|
release = threading.Event() # the test is done measuring; let the query return.
|
|
finished = threading.Event() # the search query has returned.
|
|
|
|
def slow_search(*_, **__):
|
|
entered.set()
|
|
release.wait(timeout=30)
|
|
finished.set()
|
|
return [], 0
|
|
|
|
with mock.patch('modules.videos.video.lib.search_videos', slow_search):
|
|
search_task = asyncio.create_task(
|
|
async_client.post('/api/videos/search', content=json.dumps({})))
|
|
try:
|
|
# Do not send the echo until the search request is provably inside the query.
|
|
assert await asyncio.to_thread(entered.wait, 30), 'the search request never started'
|
|
_, echo_response = await asyncio.wait_for(async_client.get('/api/echo'), timeout=30)
|
|
assert not finished.is_set(), \
|
|
'echo only completed after the search query returned; the search blocked the event loop'
|
|
finally:
|
|
release.set()
|
|
_, search_response = await search_task
|
|
|
|
assert echo_response.status_code == HTTPStatus.OK
|
|
assert search_response.status_code == HTTPStatus.OK
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_videos_default_page_pagination(test_session, video_factory, audio_factory, assert_video_search):
|
|
"""The unfiltered Videos page (no search/tags/channel) pages through every video and audio file in
|
|
date order, including files sharing a date and files without one, and reports the true total.
|
|
|
|
This page takes a fast path (one index range per mimetype instead of scanning every video), so
|
|
it must produce exactly what the generic query produces."""
|
|
# Two files share a date, one has no published date at all, and both mimetypes are mixed in.
|
|
a = video_factory(title='a', upload_date='2025-01-01', with_video_file=True)
|
|
b = video_factory(title='b', upload_date='2025-01-03', with_video_file=True)
|
|
c = audio_factory(title='c', upload_date='2025-01-03', with_info_json=True)
|
|
d = video_factory(title='d', upload_date='2025-01-02', with_video_file=True)
|
|
e = audio_factory(title='e', upload_date='2025-01-04', with_info_json=True)
|
|
f = video_factory(title='f', with_video_file=True) # no upload date
|
|
test_session.commit()
|
|
f.file_group.effective_datetime = None
|
|
test_session.commit()
|
|
test_session.refresh(f.file_group)
|
|
assert f.file_group.effective_datetime is None
|
|
|
|
from modules.videos.video import lib
|
|
with mock.patch.object(lib, '_search_videos_by_date', wraps=lib._search_videos_by_date) as fast_path:
|
|
await assert_video_search(order_by='-published_datetime', assert_total=6)
|
|
assert fast_path.called, 'the unfiltered date-ordered page should take the fast path'
|
|
fast_path.reset_mock()
|
|
await assert_video_search(order_by='-published_datetime', search_str='a', assert_total=1)
|
|
await assert_video_search(order_by='-size', assert_total=6)
|
|
assert not fast_path.called, 'searches and other orders use the generic query'
|
|
|
|
# Newest first: NULL dates last, ties broken by id ascending.
|
|
newest = [e.id, b.id, c.id, d.id, a.id, f.id]
|
|
await assert_video_search(order_by='-published_datetime', assert_total=6, assert_ids=newest)
|
|
await assert_video_search(order_by='-published_datetime', limit=2, offset=0, assert_total=6, assert_ids=newest[0:2])
|
|
await assert_video_search(order_by='-published_datetime', limit=2, offset=2, assert_total=6, assert_ids=newest[2:4])
|
|
await assert_video_search(order_by='-published_datetime', limit=2, offset=4, assert_total=6, assert_ids=newest[4:6])
|
|
# Past the end both the date fast path and the generic query report the true total.
|
|
await assert_video_search(order_by='-published_datetime', limit=2, offset=6, assert_total=6, assert_ids=[])
|
|
|
|
# Oldest first: NULL dates first (SQLite sorts NULL lowest), ties broken by id ascending.
|
|
oldest = [f.id, a.id, d.id, b.id, c.id, e.id]
|
|
await assert_video_search(order_by='published_datetime', assert_total=6, assert_ids=oldest)
|
|
await assert_video_search(order_by='published_datetime', limit=4, offset=2, assert_total=6, assert_ids=oldest[2:6])
|
|
|
|
# The fast path must return exactly what the generic query returns for the same request:
|
|
# same rows, same order (NULL placement, id tiebreaker, interleaved mimetypes), same paging.
|
|
def ids(order, limit, offset):
|
|
results, total = lib.search_videos(order=order, limit=limit, offset=offset)
|
|
return [i['id'] for i in results], total
|
|
|
|
for order in lib.INDEXED_DATE_ORDERS:
|
|
for limit, offset in ((24, 0), (2, 0), (2, 2), (4, 2), (1, 5)):
|
|
fast = ids(order, limit, offset)
|
|
with mock.patch.object(lib, 'INDEXED_DATE_ORDERS', ()):
|
|
generic = ids(order, limit, offset)
|
|
assert fast == generic, f'{order} limit={limit} offset={offset}: fast {fast} != generic {generic}'
|
|
|
|
# Generic orders used to report total=0 past the last page because COUNT(*) OVER() had no
|
|
# rows to ride on. A separate COUNT reports the true total, like the date fast path.
|
|
await assert_video_search(order_by='-download_datetime', limit=2, offset=6, assert_total=6, assert_ids=[])
|
|
await assert_video_search(order_by='-size', limit=2, offset=6, assert_total=6, assert_ids=[])
|
|
|
|
|
|
def test_cached_total(test_session):
|
|
"""Search totals are cached briefly: counting a large library is a full index scan
|
|
that costs seconds cold, and pagination can tolerate a slightly stale total."""
|
|
from wrolpi.files import lib as files_lib
|
|
calls = []
|
|
|
|
def compute():
|
|
calls.append(1)
|
|
return 42
|
|
|
|
files_lib._SEARCH_TOTALS_CACHE.clear()
|
|
with mock.patch('wrolpi.files.lib.PYTEST', False):
|
|
assert files_lib.cached_search_total('k', compute) == 42
|
|
assert files_lib.cached_search_total('k', compute) == 42
|
|
assert len(calls) == 1, 'second call within the TTL should hit the cache'
|
|
# Expire the entry.
|
|
cached_at, total = files_lib._SEARCH_TOTALS_CACHE['k']
|
|
files_lib._SEARCH_TOTALS_CACHE['k'] = (
|
|
cached_at - files_lib.SEARCH_TOTALS_CACHE_SECONDS - 1, total)
|
|
assert files_lib.cached_search_total('k', compute) == 42
|
|
assert len(calls) == 2, 'expired entry should be recomputed'
|
|
|
|
# Tests need exact totals, so the cache is bypassed under pytest.
|
|
assert files_lib.cached_search_total('k', compute) == 42
|
|
assert len(calls) == 3, 'pytest should bypass the cache'
|
|
|
|
|
|
async def test_video_description_endpoint(async_client, test_session, video_factory):
|
|
"""The description is served by its own endpoint, like comments and captions. It comes from the
|
|
indexed `c_text`, falling back to the info json on disk for a video that was never indexed with one."""
|
|
video = video_factory(with_video_file=True, with_info_json={'description': 'from the info json'})
|
|
test_session.commit()
|
|
assert video.file_group.c_text == 'from the info json', 'indexing copies the description into c_text'
|
|
|
|
request, response = await async_client.get(f'/api/videos/{video.file_group_id}/description')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json == {'description': 'from the info json'}
|
|
|
|
# A stale row (indexed before descriptions were stored) falls back to the file.
|
|
video.file_group.c_text = None
|
|
test_session.commit()
|
|
request, response = await async_client.get(f'/api/videos/{video.file_group_id}/description')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json == {'description': 'from the info json'}
|
|
|
|
request, response = await async_client.get('/api/videos/123456/description')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_json_does_not_read_info_json(async_client, test_session, video_factory, assert_video_search):
|
|
"""Video list and detail responses must never read the info json from disk.
|
|
|
|
Many videos have no description, so a fallback read from `__json__` would open and parse every
|
|
such video's info json on every list page, cold, for a field the list never displays."""
|
|
video = video_factory(with_video_file=True, with_info_json={'duration': 5}) # no description
|
|
test_session.commit()
|
|
assert not video.file_group.c_text
|
|
|
|
with mock.patch('modules.videos.models.Video.get_info_json', side_effect=AssertionError('read info json')):
|
|
await assert_video_search(assert_total=1, assert_ids=[video.id])
|
|
request, response = await async_client.get(f'/api/videos/{video.file_group_id}')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert 'description' not in response.json['file_group']['video']
|