Displaying error when video file name format is invalid.

This commit is contained in:
Roland 2024-11-29 17:01:16 -07:00
parent b7d163e7ff
commit bf29bdcff7
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
12 changed files with 201 additions and 35 deletions

View file

@ -1499,3 +1499,8 @@ export async function postRestart() {
}
return null
}
export async function postVideoFileFormat(video_file_format) {
const body = {video_file_format};
return await apiPost(`${API_URI}/videos/file_format`, body);
}

View file

@ -1607,7 +1607,7 @@ export function WarningMessage({children, size = null, icon = 'exclamation'}) {
}
export function ErrorMessage({children, size = null, icon = 'exclamation'}) {
return <Message error icon size={size}>
return <Message negative icon size={size}>
<SIcon name={icon}/>
<Message.Content>{children}</Message.Content>
</Message>

View file

@ -43,7 +43,7 @@ import {FileRowTagIcon, FilesView} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {Button, Card, Header, Loader, Placeholder, Popup, Segment, Statistic, StatisticGroup} from "./Theme";
import {deleteVideos, fetchVideoDownloaderConfig, saveVideoDownloaderConfig} from "../api";
import {deleteVideos, fetchVideoDownloaderConfig, postVideoFileFormat, saveVideoDownloaderConfig} from "../api";
import {Media, QueryContext, ThemeContext} from "../contexts/contexts";
import _ from "lodash";
import {defaultFileOrder, defaultSearchOrder} from "./Vars";
@ -222,6 +222,29 @@ function VideosPage() {
</>
}
function VideoFileNameForm({form}) {
const [message, setMessage] = React.useState(null);
const onChange = async (value) => {
const response = await postVideoFileFormat(value);
const {error, preview} = await response.json();
if (error) {
setMessage({content: error, header: 'Invalid File Name', negative: true});
} else {
setMessage({content: preview, header: 'File Name Preview', positive: true});
}
}
return <InputForm
form={form}
name='file_name_format'
path='yt_dlp_options.file_name_format'
label='Video File Format'
onChange={onChange}
message={message}
/>
}
function VideosSettings() {
useTitle('Videos Settings');
@ -255,62 +278,55 @@ function VideosSettings() {
<Form>
<Grid>
<Grid.Row>
<Grid.Column>
<Grid.Row columns={2}>
<Grid.Column mobile={16} computer={8}>
<VideoResolutionSelectorForm
form={form}
name='video_resolutions'
path='video_resolutions'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<InputForm
form={form}
name='file_name_format'
path='yt_dlp_options.file_name_format'
label='Video File Format'
/>
<Grid.Column mobile={16} computer={8}>
<VideoFileNameForm form={form}/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<Grid.Row columns={2}>
<Grid.Column mobile={16} computer={8}>
<ToggleForm
form={form}
label='Do not overwrite existing files'
name='nooverwrites'
path='yt_dlp_options.nooverwrites'
icon='file video'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<Grid.Column mobile={16} computer={8}>
<ToggleForm
form={form}
label='Download automatic subtitles'
name='writeautomaticsub'
path='yt_dlp_options.writeautomaticsub'
icon='closed captioning'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<Grid.Column mobile={16} computer={8}>
<ToggleForm
form={form}
label='Download subtitles'
name='writesubtitles'
path='yt_dlp_options.writesubtitles'
icon='closed captioning outline'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<Grid.Column mobile={16} computer={8}>
<ToggleForm
form={form}
label='Download thumbnail'
name='writethumbnail'
path='yt_dlp_options.writethumbnail'
icon='image'
/>
</Grid.Column>
</Grid.Row>

View file

@ -1,7 +1,8 @@
import React from "react";
import _ from "lodash";
import {HelpPopup, RequiredAsterisk, Toggle, validURL, validURLs} from "../components/Common";
import {FormInput, TextArea} from "../components/Theme";
import {FormInput, Icon, TextArea} from "../components/Theme";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
async function asyncNoOp() {
@ -158,6 +159,8 @@ export function useForm({
// Trigger validation after a delay
debouncedValidate(path, value);
return value;
}
const addValidator = (path, validator) => {
@ -207,7 +210,7 @@ export function useForm({
return [inputProps, inputAttrs]
}
const getInputProps = ({name, validator, path, required = false, type = 'text'}) => {
const getInputProps = ({name, validator, path, required = false, type = 'text', onChange = null}) => {
// Props for <input/>
path = path || name;
@ -218,11 +221,19 @@ export function useForm({
const [customProps, inputAttrs] = getCustomProps({name, validator, path, type, required});
const localHandleInputEvent = async (e) => {
if (e) e.preventDefault();
const value = handleInputEvent(e);
if (onChange) {
await onChange(value);
}
}
// Attributes that should be passed as properties to the input.
const inputProps = {
...customProps,
name: name,
onChange: handleInputEvent,
onChange: localHandleInputEvent,
error: errors[path] || null,
required: required ? null : undefined,
}
@ -280,10 +291,21 @@ export function InputForm({
helpPosition = 'top',
extraInputProps = {},
disabled = false,
onChange = null,
message = null,
}) {
const [inputProps, inputAttrs] = form.getInputProps({name, path, validator, type, required});
const [inputProps, inputAttrs] = form.getInputProps({name, path, validator, type, required, onChange});
inputProps.disabled = disabled || inputProps.disabled;
let messageElm = null;
if (message && message.positive) {
messageElm = <Message {...message} positive/>;
} else if (message && message.negative) {
messageElm = <Message {...message}/>;
} else if (message) {
messageElm = <Message {...message}/>;
}
return <>
<label for={`${name}_input`}>
<b>{label} {required && <RequiredAsterisk/>}</b>
@ -301,6 +323,7 @@ export function InputForm({
>
<input {...extraInputProps} {...inputProps}/>
</FormInput>
{messageElm}
</>
}
@ -403,11 +426,13 @@ export function UrlsTextarea({name = 'urls', required, form}) {
</FormInput>
}
export function ToggleForm({form, name, label, path}) {
export function ToggleForm({form, name, label, path, icon = null, iconSize = 'big'}) {
const [inputProps, inputAttrs] = form.getCustomProps({name, path})
const iconElm = icon ? <Icon size={iconSize} name={icon}/> : null;
return <FormInput
label={label}>
{iconElm}
<Toggle
disabled={form.disabled}
name={name}

View file

@ -9,6 +9,7 @@ from wrolpi.api_utils import json_response
from wrolpi.common import logger
from . import lib, schema
from .channel.api import channel_bp
from .downloader import preview_filename
from .lib import format_videos_destination
from .models import Channel
from .video.api import video_bp
@ -45,3 +46,23 @@ async def channel_tag_info(_: Request, body: schema.ChannelTagInfoRequest):
videos_destination = format_videos_destination(channel_name, tag_name, channel_url)
ret = dict(videos_destination=videos_destination)
return json_response(ret)
@content_bp.post('/file_format')
@openapi.definition(
description='Preview the video file format',
body=schema.VideoFileFormatRequest,
)
@validate(schema.VideoFileFormatRequest)
async def post_file_format(_: Request, body: schema.VideoFileFormatRequest):
try:
preview = preview_filename(body.video_file_format)
return json_response(dict(
video_file_format=body.video_file_format,
preview=preview,
))
except RuntimeError as e:
return json_response(dict(
error=str(e),
video_file_format=body.video_file_format,
), status=HTTPStatus.BAD_REQUEST)

View file

@ -1,4 +1,5 @@
#! /usr/bin/env python3
import copy
import json
import logging
import os.path
@ -87,6 +88,31 @@ def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
return f'{dir_name}/{file_name}'
def preview_filename(filename_format: str) -> str:
"""Return an example video file name formatted using the provided filename format.
@raise RuntimeError: If the format is invalid.
"""
if not filename_format.endswith('.%(ext)s'):
raise RuntimeError('Filename must end with .%(ext)s')
options = get_videos_downloader_config().yt_dlp_options
options['outtmpl'] = filename_format
ydl = YoutubeDL(copy.deepcopy(options))
entry = dict(
uploader='WROLPi',
timestamp=int(now().timestamp()),
upload_date=now().strftime('%Y%m%d'),
id='Qz-FuenRylQ',
title='The title of the video',
ext='mp4',
description='A description of the video',
)
filename = prepare_filename(entry, ydl=ydl)[1:]
return filename
@cached_multiprocessing_result
async def fetch_video_duration(url: str) -> int:
"""Get video duration in seconds. Attempts to get the duration from a Video that has already been downloaded,
@ -583,7 +609,7 @@ class VideoDownloader(Downloader, ABC):
options['cachdir'] = YTDLP_CACHE_DIR
# Create a new YoutubeDL for the output directory.
ydl = YoutubeDL(options)
ydl = YoutubeDL(copy.deepcopy(options))
ydl.params['logger'] = ydl_logger
ydl.add_default_info_extractors()
@ -615,7 +641,7 @@ class VideoDownloader(Downloader, ABC):
# Get entry info json.
options['outtmpl'] = str(final_filename)
ydl = YoutubeDL(options)
ydl = YoutubeDL(copy.deepcopy(options))
ydl.params['logger'] = ydl_logger
ydl.add_default_info_extractors()
entry = extract_info(url, ydl=ydl, process=True)

View file

@ -44,6 +44,7 @@ class VideoInfoJSON:
channel_url: str | None = None
duration: int | None = None
epoch: int | None = None
timestamp: datetime | None = None
title: str | None = None
upload_date: datetime | None = None
url: str | None = None
@ -62,11 +63,17 @@ def extract_video_info_json(video: Video) -> VideoInfoJSON:
upload_date = dates.strpdate(i) if (i := info_json.get('upload_date')) else None
if upload_date:
upload_date = upload_date.astimezone(pytz.UTC)
timestamp = None
try:
timestamp = dates.from_timestamp(info_json['timestamp']) if info_json.get('timestamp') else None
except Exception:
pass
video_info_json.channel_source_id = info_json.get('channel_id') or info_json.get('uploader_id') or None
video_info_json.channel_url = info_json.get('channel_url') or info_json.get('uploader_url') or None
video_info_json.duration = int(i) if (i := info_json.get('duration')) else None
video_info_json.epoch = int(i) if (i := info_json.get('epoch')) else None
video_info_json.timestamp = timestamp
video_info_json.upload_date = upload_date
video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None
video_info_json.view_count = int(i) if (i := info_json.get('view_count')) else None
@ -95,8 +102,11 @@ def validate_video(video: Video, channel_generate_poster: bool):
video.file_group.title = video_info_json.title
video.file_group.length = video_info_json.duration
video.file_group.url = video_info_json.url
json_published_datetime = video.file_group.published_datetime = \
video_info_json.upload_date or video.file_group.published_datetime
json_published_datetime = video.file_group.published_datetime = (
video_info_json.timestamp # The exact second the video was published.
or video_info_json.upload_date # The day the video was published.
or video.file_group.published_datetime
)
video.file_group.download_datetime = from_timestamp(video_info_json.epoch) if video_info_json.epoch else None
# View count will probably be overwritten by more recent data when this Video's Channel is
# updated.
@ -349,12 +359,20 @@ class VideoDownloaderConfigYtDlpOptionsValidator:
file_name_format: str
nooverwrites: bool
quiet: bool
merge_output_format: str
writeautomaticsub: bool
writeinfojson: bool
writesubtitles: bool
writethumbnail: bool
youtube_include_dash_manifest: bool
def __post_init__(self):
from modules.videos.downloader import preview_filename
try:
preview_filename(self.file_name_format)
except Exception as e:
raise ValueError(f'file_name_format is invalid: {str(e)}')
@dataclass
class VideoDownloaderConfigValidator:
@ -362,6 +380,12 @@ class VideoDownloaderConfigValidator:
version: int = None
yt_dlp_options: VideoDownloaderConfigYtDlpOptionsValidator = field(default_factory=dict)
def __post_init__(self):
allowed_fields = {i.name for i in dataclasses.fields(VideoDownloaderConfigYtDlpOptionsValidator)}
yt_dlp_options = {k: v for k, v in dict(self.yt_dlp_options).items() if k in allowed_fields}
VideoDownloaderConfigYtDlpOptionsValidator(**yt_dlp_options)
self.yt_dlp_options = yt_dlp_options
class VideoDownloaderConfig(ConfigFile):
file_name = 'videos_downloader.yaml'

View file

@ -229,3 +229,8 @@ class CensoredVideoResponse:
@dataclass
class ChannelSearchRequest:
tag_names: List[str] = field(default_factory=list)
@dataclass
class VideoFileFormatRequest:
video_file_format: str

View file

@ -307,3 +307,21 @@ async def test_format_videos_description(async_client, test_session, test_direct
request, response = await async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name'
@pytest.mark.asyncio
async def test_video_file_format(async_client, test_session, fake_now):
body = dict(video_file_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s')
request, response = await async_client.post('/api/videos/file_format', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['preview'] == 'WROLPi_20000101_Qz-FuenRylQ_The title of the video.mp4'
body = dict(video_file_format='%(upload_date)s_%(id)s_%(title)s.%(ext)s')
request, response = await async_client.post('/api/videos/file_format', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['preview'] == '20000101_Qz-FuenRylQ_The title of the video.mp4'
body = dict(video_file_format='%(upload_date)s_%(id)s_%(title)s')
request, response = await async_client.post('/api/videos/file_format', json=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json['error'] == 'Filename must end with .%(ext)s'

View file

@ -8,7 +8,7 @@ from unittest import mock
import pytest
from modules.videos.downloader import VideoDownloader, \
get_or_create_channel, channel_downloader, video_downloader
get_or_create_channel, channel_downloader, video_downloader, preview_filename
from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory
from wrolpi.downloader import Download, DownloadResult
@ -494,3 +494,19 @@ async def test_channel_download_crud(test_session, async_client, assert_download
assert download.settings['video_count_limit'] == 100
assert download.settings['video_resolutions'] == ['1080p', '720p', '480p', '360p', 'maximum']
assert download.settings['video_format'] == 'mp4'
def test_preview_filename(test_directory, fake_now):
assert preview_filename('%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s') \
== 'WROLPi_20000101_Qz-FuenRylQ_The title of the video.mp4'
assert preview_filename('%(upload_date)s_%(id)s_%(title)s.%(ext)s') \
== '20000101_Qz-FuenRylQ_The title of the video.mp4'
assert preview_filename('%(upload_date)s_%(title)s.%(ext)s') \
== '20000101_The title of the video.mp4'
assert preview_filename('%(upload_date)s_%(description)s.%(ext)s') \
== '20000101_A description of the video.mp4'
assert preview_filename('%(timestamp)s_%(title)s.%(ext)s') \
== '946684800_The title of the video.mp4'
with pytest.raises(RuntimeError):
preview_filename('%(upload_date)s_%(title)s.asdf')

View file

@ -139,7 +139,7 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
@pytest.mark.asyncio
async def test_orphaned_files(test_session, make_files_structure, test_directory, video_factory):
async def test_orphaned_files(async_client, test_session, make_files_structure, test_directory, video_factory):
# A Video without associated files is not orphaned.
vid1 = video_factory(title='vid1', with_video_file=True)
# The video files will be removed...
@ -271,6 +271,7 @@ async def test_format_videos_destination(async_client, test_directory):
@pytest.mark.asyncio
async def test_videos_downloader_config_api(async_client, test_directory, test_videos_downloader_config):
# The WROLPi default resolutions.
config = lib.get_videos_downloader_config()
assert config.video_resolutions == ['1080p', '720p', '480p', 'maximum']
@ -278,11 +279,12 @@ async def test_videos_downloader_config_api(async_client, test_directory, test_v
assert response.status == HTTPStatus.OK
config = response.json['config']
# Change the `video_resolutions`
config['video_resolutions'] = ['720p', 'maximum']
body = dict(config=config)
request, response = await async_client.post('/api/config/videos_downloader.yaml', json=body)
assert response.status == HTTPStatus.NO_CONTENT
# Config file was actually changed.
config = lib.get_videos_downloader_config()
assert config.video_resolutions == ['720p', 'maximum']
assert (test_directory / 'config/videos_downloader.yaml').is_file()
@ -290,3 +292,11 @@ async def test_videos_downloader_config_api(async_client, test_directory, test_v
contents = config.read_config_file()
assert contents['video_resolutions'] == ['720p', 'maximum']
assert '1080p' not in config.get_file().read_text()
# Cannot set invalid `file_name_format`.
config = lib.get_videos_downloader_config().dict()
config['yt_dlp_options']['file_name_format'] = 'invalid format'
body = dict(config=config)
request, response = await async_client.post('/api/config/videos_downloader.yaml', json=body)
assert response.status == HTTPStatus.BAD_REQUEST
assert 'Invalid config' in response.json.get('error')

View file

@ -77,7 +77,7 @@ async def test_video_channel_refresh(async_client, test_session, test_directory,
@pytest.mark.asyncio
async def test_delete_duplicate_video(test_session, channel_factory, video_factory, tag_factory):
async def test_delete_duplicate_video(async_client, test_session, channel_factory, video_factory, tag_factory):
"""If duplicate Video's exist, and everything about the files matches, delete a random one."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'