Adding feature where Videos can be downloaded to a destination directory.

This commit is contained in:
Roland 2022-11-09 08:33:55 -07:00
parent 61d2ffde36
commit f1bc29d76f
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
22 changed files with 248 additions and 181 deletions

View file

@ -2,7 +2,7 @@ import {LoadStatistic, PageContainer, SearchInput, useTitle} from "./components/
import {useSearchFiles} from "./hooks/customHooks";
import React, {useContext, useState} from "react";
import {StatusContext} from "./contexts/contexts";
import {DownloadMenu} from "./components/Upload";
import {DownloadMenu} from "./components/Download";
import {FilesSearchView} from "./components/Files";
import {Header, Segment, Statistic, StatisticGroup} from "./components/Theme";
import {Link} from "react-router-dom";

View file

@ -171,9 +171,9 @@ export async function deleteVideos(videoIds) {
export async function getDirectories(search_str) {
let form_data = {'search_str': search_str || null};
let response = await apiPost(`${VIDEOS_API}/directories`, form_data);
let response = await apiPost(`${API_URI}/files/directories`, form_data);
if (response.status === 200) {
return (await response.json())['directories'];
return await response.json();
}
return [];
}
@ -461,8 +461,14 @@ export async function getArchive(archiveId) {
}
}
export async function postDownload(urls, downloader, frequency, sub_downloader, excludedURLs) {
let body = {urls: urls, downloader: downloader, frequency: frequency || null, excluded_urls: excludedURLs};
export async function postDownload(urls, downloader, frequency, sub_downloader, excludedURLs, destination) {
let body = {
urls: urls,
downloader: downloader,
frequency: frequency || null,
excluded_urls: excludedURLs,
destination: destination || null,
};
if (sub_downloader) {
body['sub_downloader'] = sub_downloader;
}

View file

@ -16,8 +16,10 @@ import {
import {createChannel, deleteChannel, downloadChannel, refreshChannel, updateChannel, validateRegex} from "../api";
import Container from "semantic-ui-react/dist/commonjs/elements/Container";
import {
DirectoryInput,
frequencyOptions,
humanFileSize, isEmpty,
humanFileSize,
isEmpty,
RequiredAsterisk,
secondsToDate,
secondsToFrequency,
@ -30,7 +32,7 @@ import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import Confirm from "semantic-ui-react/dist/commonjs/addons/Confirm";
import {ChannelPlaceholder} from "./Placeholder";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {useChannel, useChannels, useDirectories, useSettings} from "../hooks/customHooks";
import {useChannel, useChannels} from "../hooks/customHooks";
import {toast} from "react-semantic-toasts";
import _ from "lodash";
import {
@ -49,41 +51,6 @@ import {
import Dropdown from "semantic-ui-react/dist/commonjs/modules/Dropdown";
function DirectoryInput({disabled, error, placeholder, setInput, value, required}) {
const {directory, directories, setDirectory} = useDirectories(value);
const {settings} = useSettings();
if (!directories || !settings) {
return <></>;
}
const localSetInput = (e, {value}) => {
setInput(value);
setDirectory(value);
}
const {media_directory} = settings;
return (
<div>
<Input
required={required}
disabled={disabled}
name='directory'
list='directories'
error={error}
label={media_directory}
value={directory}
onChange={localSetInput}
placeholder={placeholder}
/>
<datalist id='directories'>
{directories.map(i => <option key={i} value={i}>{i}</option>)}
</datalist>
</div>
);
}
function ChannelStatistics({statistics}) {
if (!statistics) {
return <></>

View file

@ -2,7 +2,7 @@ import React, {useContext, useEffect, useState} from "react";
import {Card, Container, IconGroup, Image, Input, Modal, Pagination, Responsive} from 'semantic-ui-react';
import {Link, NavLink} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useDownloaders, useHotspot, useThrottle} from "../hooks/customHooks";
import {useDirectories, useDownloaders, useHotspot, useSettings, useThrottle} from "../hooks/customHooks";
import {StatusContext, ThemeContext} from "../contexts/contexts";
import {Button, darkTheme, Form, Header, Icon, lightTheme, Menu, Popup, Statistic} from "./Theme";
@ -702,3 +702,41 @@ export function useTitle(title) {
}
}, [title]);
}
export function DirectoryInput({disabled, error, placeholder, setInput, value, required, isDirectory}) {
const {directory, directories, setDirectory, isDir} = useDirectories(value);
const {settings} = useSettings();
if (!directories || !settings) {
return <></>;
}
const localSetInput = (e, {value}) => {
setInput(value);
setDirectory(value);
}
const {media_directory} = settings;
error = error || isDirectory && !isDir;
return (<div>
<Input
action={{
color: error ? 'red' : 'green',
labelPosition: 'left',
icon: 'folder',
content: media_directory,
}}
required={required}
disabled={disabled}
actionPosition='left'
value={directory}
onChange={localSetInput}
placeholder={placeholder}
list='directories'
/>
<datalist id='directories'>
{directories.map(i => <option key={i} value={i}>{i}</option>)}
</datalist>
</div>);
}

View file

@ -55,7 +55,7 @@ function CoinRow({header, qrCodeValue, address, buttonColor}) {
export function Donate() {
useTitle('Donate');
return (<PageContainer>
return <PageContainer>
<Segment>
<Header as='h1'>Donate</Header>
<Header as='h3'>We appreciate any support you can provide to WROLPi!</Header>
@ -64,5 +64,5 @@ export function Donate() {
<CoinRow header='Ethereum' qrCodeValue={ethereumContent} address={ethereumAddress} buttonColor='blue'/>
<CoinRow header='Bitcoin' qrCodeValue={bitcoinContent} address={bitcoinAddress} buttonColor='yellow'/>
</Segment>
</PageContainer>);
</PageContainer>;
}

View file

@ -1,10 +1,10 @@
import React, {useContext, useState} from "react";
import {getDownloaders, postDownload} from "../api";
import {frequencyOptions, HelpPopup, rssFrequencyOptions, WROLModeMessage} from "./Common";
import {DirectoryInput, frequencyOptions, HelpPopup, rssFrequencyOptions, WROLModeMessage} from "./Common";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {ThemeContext} from "../contexts/contexts";
import {Accordion, Button, Form, FormInput, Header, Loader, TextArea} from "./Theme";
import {Accordion, Button, Form, FormField, FormGroup, FormInput, Header, Loader, Segment, TextArea} from "./Theme";
import {AccordionContent, AccordionTitle, FormDropdown} from "semantic-ui-react";
const validUrl = /^(http|https):\/\/[^ "]+$/;
@ -13,16 +13,27 @@ class Downloader extends React.Component {
constructor(props) {
super(props);
this.state = {
urls: '', valid: true, pending: false, downloader: props.downloader,
advancedOpen: false,
destination: '',
downloader: props.downloader,
pending: false,
urls: '',
valid: true,
};
}
submitDownload = async () => {
let {urls, downloader} = this.state;
let {urls, downloader, destination} = this.state;
if (urls) {
this.setState({pending: true});
try {
let response = await postDownload(urls, downloader);
let response = await postDownload(
urls,
downloader,
null,
null,
null,
destination);
if (response.status === 204) {
this.setState({urls: '', pending: false});
}
@ -52,7 +63,29 @@ class Downloader extends React.Component {
render() {
let disabled = !this.state.urls || !this.state.valid || this.state.pending || !this.state.downloader;
const {header} = this.props;
const {advancedOpen, destination} = this.state;
const {header, withDestination} = this.props;
const advancedAccordion = <Accordion>
<AccordionTitle onClick={() => this.setState({advancedOpen: !advancedOpen})}>
<Icon name='dropdown'/>Advanced
</AccordionTitle>
<AccordionContent active={advancedOpen}>
<Segment>
<FormGroup>
<FormField width={16}>
<label>Destination</label>
<DirectoryInput
isDirectory={true}
value={destination}
setInput={value => this.handleInputChange('directory', value)}
placeholder='download/into/custom/directory'
/>
</FormField>
</FormGroup>
</Segment>
</AccordionContent>
</Accordion>;
return <ThemeContext.Consumer>
{({i}) => (<Form onSubmit={this.submitDownload}>
@ -64,6 +97,7 @@ class Downloader extends React.Component {
onChange={this.handleInputChange}
value={this.state.urls}
/>
{withDestination && advancedAccordion}
<Button content='Cancel' onClick={this.props.clearSelected}/>
<Button primary style={{marginTop: '1em'}} disabled={disabled}>Download</Button>
</Form>)}
@ -333,7 +367,7 @@ export function DownloadMenu({onOpen}) {
'archive': <Downloader clearSelected={clearSelected} header={<><Icon name='file alternate'/> Archive</>}
downloader='archive'/>,
'video': <Downloader clearSelected={clearSelected} header={<><Icon name='video'/> Videos</>}
downloader='video'/>,
downloader='video' withDestination={true}/>,
'video_channel': <ChannelDownload clearSelected={clearSelected}/>,
'rss': <RSSDownload clearSelected={clearSelected} header={<><Icon name='rss square'/> RSS</>}/>,
};

View file

@ -576,13 +576,20 @@ export const useThrottle = () => {
export const useDirectories = (defaultDirectory) => {
const [directories, setDirectories] = useState();
const [directory, setDirectory] = useState(defaultDirectory);
const [exists, setExists] = useState();
const [isDir, setIsDir] = useState();
const [isFile, setIsFile] = useState();
const fetchDirectories = async () => {
if (defaultDirectory && directory === '') {
setDirectory(defaultDirectory);
}
try {
setDirectories(await getDirectories(directory));
const {directories, is_dir, exists_, is_file} = await getDirectories(directory);
setDirectories(directories);
setIsDir(is_dir);
setExists(exists_);
setIsFile(is_file);
} catch (e) {
toast({
type: 'error',
@ -597,7 +604,7 @@ export const useDirectories = (defaultDirectory) => {
fetchDirectories();
}, [directory, defaultDirectory]);
return {directory, directories, setDirectory};
return {directory, directories, setDirectory, exists, isDir, isFile};
}
export const useSettings = () => {

View file

@ -8,7 +8,7 @@ from datetime import timedelta
from decimal import Decimal
from functools import partial
from pathlib import Path
from typing import Union, List, Set, Iterable, Optional, Tuple
from typing import List, Set, Iterable, Optional, Tuple
import PIL
from PIL import Image
@ -136,35 +136,6 @@ def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None,
raise ChannelSourceIdConflict()
def get_matching_directories(path: Union[str, Path]) -> List[str]:
"""
Return a list of directory strings that start with the provided path. If the path is a directory, return it's
subdirectories, if the directory contains no subdirectories, return the directory.
"""
path = str(path)
ignored_directories = {
str(get_no_channel_directory()),
}
if os.path.isdir(path):
# The provided path is a directory, return its subdirectories, or itself if no subdirectories exist
paths = [os.path.join(path, i) for i in os.listdir(path)]
paths = sorted(i for i in paths if os.path.isdir(i) and i not in ignored_directories)
if len(paths) == 0:
return [path]
return paths
head, tail = os.path.split(path)
paths = os.listdir(head)
paths = [os.path.join(head, i) for i in paths]
pattern = path.lower()
paths = sorted(
i for i in paths if os.path.isdir(i) and i.lower().startswith(pattern) and i not in ignored_directories)
return paths
def duration_to_seconds(duration: str) -> int:
"""Convert ffmpeg duration to seconds"""
hours, minutes, seconds = duration.split(':')

View file

@ -256,9 +256,12 @@ class VideoDownloader(Downloader, ABC):
logger.info('Could not find channel')
# Use the default directory if this video has no channel.
out_dir = get_no_channel_directory()
if channel:
out_dir = channel.directory
if settings.get('destination'):
out_dir = pathlib.Path(settings['destination'])
else:
out_dir = get_no_channel_directory()
if channel:
out_dir = channel.directory
out_dir.mkdir(exist_ok=True, parents=True)
logs = None # noqa

View file

@ -157,16 +157,6 @@ class PaginationQuery:
offset: int
@dataclass
class DirectoriesRequest:
search_str: Optional[str] = ''
@dataclass
class DirectoriesResponse:
directories: List[str]
@dataclass
class FavoriteRequest:
video_id: int

View file

@ -12,49 +12,13 @@ from modules.videos.models import Channel, Video
from wrolpi.common import get_absolute_media_path, sanitize_link
from wrolpi.dates import local_timezone, now
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.test.common import build_test_directories, TestAPI
from wrolpi.vars import PROJECT_DIR
from .. import common
from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \
from ..common import convert_image, remove_duplicate_video_paths, \
apply_info_json, get_video_duration, generate_video_poster, is_valid_poster
from ..lib import save_channels_config, get_channels_config, import_channels_config
class TestCommon(TestAPI):
def test_matching_directories(self):
structure = [
'foo/qux/',
'Bar/',
'baz/baz'
'barr',
'bazz',
]
with build_test_directories(structure) as temp_dir:
temp_dir = pathlib.Path(temp_dir)
# No directories have c
matches = get_matching_directories(temp_dir / 'c')
assert matches == []
# Get all directories starting with f
matches = get_matching_directories(temp_dir / 'f')
assert matches == [str(temp_dir / 'foo')]
# Get all directories starting with b, ignore case
matches = get_matching_directories(temp_dir / 'b')
assert matches == [str(temp_dir / 'Bar'), str(temp_dir / 'baz')]
# baz matches, but it has no subdirectories
matches = get_matching_directories(temp_dir / 'baz')
assert matches == [str(temp_dir / 'baz')]
# foo is an exact match, return subdirectories
matches = get_matching_directories(temp_dir / 'foo')
assert matches == [str(temp_dir / 'foo/qux')]
def test_get_absolute_media_path():
path = get_absolute_media_path('videos')
assert str(path).endswith('videos')

View file

@ -443,6 +443,21 @@ async def test_download_result(test_session, test_directory, video_download_mana
assert video.poster_file.associated, 'Poster file was not associated.'
@pytest.mark.asyncio
async def test_download_destination(test_session, test_directory, video_download_manager,
mock_video_process_runner, mock_video_extract_info, mock_video_prepare_filename):
mock_video_extract_info.return_value = example_video_json
settings = dict(destination=f'{test_directory}/custom')
video_download_manager.create_download('https://example.com', downloader=VideoDownloader.name, settings=settings)
mock_video_process_runner.return_value = (0, {'stdout': '', 'stderr': ''})
mock_video_prepare_filename.return_value = str(test_directory / 'test video.mp4')
await video_download_manager.wait_for_all_downloads()
# Output directory matches the custom directory specified.
assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default']\
.startswith(f'{test_directory}/custom/%(uploader)s')
example_playlist_json = {
'_type': 'playlist',
'availability': None,

View file

@ -5,13 +5,12 @@ from sanic.request import Request
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import logger, wrol_mode_check, run_after, get_media_directory, \
get_relative_to_media_directory
from wrolpi.common import logger, wrol_mode_check, run_after
from wrolpi.errors import InvalidOrderBy, ValidationError
from wrolpi.root_api import json_response
from wrolpi.schema import JSONErrorResponse
from . import lib
from .. import common, schema
from .. import schema
video_bp = Blueprint('Video', '/api/videos')
@ -52,21 +51,6 @@ async def search(_: Request, body: schema.VideoSearchRequest):
return json_response(ret)
@video_bp.post('/directories')
@openapi.definition(
summary='Get all directories that match the search_str, prefixed by the media directory.',
body=schema.DirectoriesRequest,
)
@openapi.response(HTTPStatus.OK, schema.DirectoriesResponse)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@validate(schema.DirectoriesRequest)
def directories(_, body: schema.DirectoriesRequest):
search_str = str(get_media_directory() / (body.search_str or ''))
dirs = common.get_matching_directories(search_str)
dirs = [str(get_relative_to_media_directory(i)) for i in dirs]
return response.json({'directories': dirs})
@video_bp.delete('/video/<video_ids:[0-9,]+>')
@video_bp.delete('/video/<video_ids:int>')
@openapi.description('Delete videos.')

View file

@ -157,28 +157,3 @@ def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixtu
wrol_mode_fixture(False)
assert not test_download_manager.stopped.is_set()
def test_video_directory_search(test_client, make_files_structure):
"""Test that video directories can be searched."""
make_files_structure([
'foo/',
'fool/',
'not a directory',
])
def assert_directories(search_str, expected):
body = dict(search_str=search_str)
request, response = test_client.post('/api/videos/directories', content=dumps(body))
assert response.status_code == HTTPStatus.OK
assert response.json['directories'] == expected
# All directories are returned.
assert_directories(None, ['foo', 'fool', 'videos'])
assert_directories('', ['foo', 'fool', 'videos'])
# Matches both directories.
assert_directories('fo', ['foo', 'fool'])
# Matches the one directory exactly.
assert_directories('foo', ['foo'])
# Does not exist.
assert_directories('food', [])

View file

@ -241,7 +241,7 @@ def make_files_structure(test_directory) -> Callable[[Union[List, Dict]], List[p
for name in paths_:
path = test_directory / name
if name.endswith('/'):
path.mkdir()
path.mkdir(parents=True, exist_ok=True)
else:
path.parent.mkdir(parents=True, exist_ok=True)
path.touch()

View file

@ -1,5 +1,5 @@
import asyncio
import pathlib
from http import HTTPStatus
from typing import List
from urllib.request import Request
@ -7,10 +7,11 @@ from sanic import response
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import get_media_directory, wrol_mode_check, background_task
from wrolpi.common import get_media_directory, wrol_mode_check, background_task, get_relative_to_media_directory
from wrolpi.errors import InvalidFile
from wrolpi.root_api import get_blueprint, json_response
from . import lib, schema
from ..schema import JSONErrorResponse
from ..vars import PYTEST
bp = get_blueprint('Files', '/api/files')
@ -101,3 +102,21 @@ async def refresh_directory(_: Request, body: schema.DirectoryRefreshRequest):
async def post_search_files(_: Request, body: schema.FilesSearchRequest):
files, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetype, body.model)
return json_response(dict(files=files, totals=dict(files=total)))
@bp.post('/directories')
@openapi.definition(
summary='Get all directories that match the search_str, prefixed by the media directory.',
body=schema.DirectoriesRequest,
)
@openapi.response(HTTPStatus.OK, schema.DirectoriesResponse)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@validate(schema.DirectoriesRequest)
def directories(_, body: schema.DirectoriesRequest):
path = get_media_directory() / (body.search_str or '')
search_str = str(path)
dirs = lib.get_matching_directories(search_str)
dirs = [str(get_relative_to_media_directory(i)) for i in dirs]
body = {'directories': dirs, 'exists': path.exists(), 'is_dir': path.is_dir(), 'is_file': path.is_file()}
return response.json(body)

View file

@ -1,6 +1,7 @@
import asyncio
import datetime
import glob
import os
import pathlib
import re
import subprocess
@ -423,3 +424,30 @@ def assign_file_prefetched_models(files: List[File]) -> List[File]:
pass
return files
def get_matching_directories(path: Union[str, Path]) -> List[str]:
"""
Return a list of directory strings that start with the provided path. If the path is a directory, return it's
subdirectories, if the directory contains no subdirectories, return the directory.
"""
path = str(path)
ignored_directories = {}
if os.path.isdir(path):
# The provided path is a directory, return its subdirectories, or itself if no subdirectories exist
paths = [os.path.join(path, i) for i in os.listdir(path)]
paths = sorted(i for i in paths if os.path.isdir(i) and i not in ignored_directories)
if len(paths) == 0:
return [path]
return paths
head, tail = os.path.split(path)
paths = os.listdir(head)
paths = [os.path.join(head, i) for i in paths]
pattern = path.lower()
paths = sorted(
i for i in paths if os.path.isdir(i) and i.lower().startswith(pattern) and i not in ignored_directories)
return paths

View file

@ -24,3 +24,13 @@ class FilesSearchRequest:
@dataclass
class DirectoryRefreshRequest:
directory: str = None
@dataclass
class DirectoriesRequest:
search_str: Optional[str] = ''
@dataclass
class DirectoriesResponse:
directories: List[str]

View file

@ -206,3 +206,28 @@ def test_associated_files(test_session, test_client, make_files_structure):
{'path': 'video.mp4'},
]
)
def test_directory_search(test_client, make_files_structure):
"""Test that directories can be searched."""
make_files_structure([
'foo/',
'fool/',
'not a directory',
])
def assert_directories(search_str, expected):
body = dict(search_str=search_str)
request, response = test_client.post('/api/files/directories', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK
assert response.json['directories'] == expected
# All directories are returned.
assert_directories(None, ['foo', 'fool'])
assert_directories('', ['foo', 'fool'])
# Matches both directories.
assert_directories('fo', ['foo', 'fool'])
# Matches the one directory exactly.
assert_directories('foo', ['foo'])
# Does not exist.
assert_directories('food', [])

View file

@ -396,3 +396,33 @@ def test_glob_shared_stem(make_files_structure):
check(vid2, ['video2 [name].mp4', 'video2 [name].info.json'])
check(vid2j, ['video2 [name].mp4', 'video2 [name].info.json'])
def test_matching_directories(make_files_structure, test_directory):
make_files_structure([
'foo/qux/',
'Bar/',
'baz/baz'
'barr',
'bazz',
])
# No directories have c
matches = lib.get_matching_directories(test_directory / 'c')
assert matches == []
# Get all directories starting with f
matches = lib.get_matching_directories(test_directory / 'f')
assert matches == [str(test_directory / 'foo')]
# Get all directories starting with b, ignore case
matches = lib.get_matching_directories(test_directory / 'b')
assert matches == [str(test_directory / 'Bar'), str(test_directory / 'baz')]
# baz matches, but it has no subdirectories
matches = lib.get_matching_directories(test_directory / 'baz')
assert matches == [str(test_directory / 'baz')]
# foo is an exact match, return subdirectories
matches = lib.get_matching_directories(test_directory / 'foo')
assert matches == [str(test_directory / 'foo/qux')]

View file

@ -221,7 +221,7 @@ async def post_download(_: Request, body: DownloadRequest):
downloader = body.downloader
if not downloader or downloader in ('auto', 'None', 'null'):
downloader = None
settings = dict(excluded_urls=excluded_urls)
settings = dict(excluded_urls=excluded_urls, destination=body.destination)
if body.frequency:
download_manager.recurring_download(urls[0], body.frequency, downloader=downloader,
sub_downloader=body.sub_downloader, reset_attempts=True,

View file

@ -54,6 +54,7 @@ class DownloadRequest:
frequency: Optional[int] = None
sub_downloader: Optional[str] = None
excluded_urls: Optional[str] = None
destination: Optional[str] = None
@dataclass