Adding edit form for Scrape downloader.

This commit is contained in:
Roland 2024-12-03 12:44:38 -07:00
parent bf29bdcff7
commit fe521c99f3
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
17 changed files with 314 additions and 262 deletions

View file

@ -413,74 +413,6 @@ export async function getVideosStatistics() {
return response
}
export async function createChannelDownload(channelId, url, frequency, title_include, title_exclude, tag_names) {
const body = {
url: url,
frequency: frequency,
settings: {
title_include: title_include,
title_exclude: title_exclude,
tag_names: tag_names,
},
}
url = `${VIDEOS_API}/channels/${channelId}/download`;
const response = await apiPost(url, body);
if (!response.ok) {
const json = await response.json();
if (json['code'] === 'INVALID_DOWNLOAD') {
toast({
type: 'error',
title: 'Cannot Download!',
description: 'This channel does not have a download record. Modify the frequency.',
time: 5000,
});
} else {
const message = await getErrorMessage(response, 'Could not create Channel download.');
toast({
type: 'error',
title: 'Creating Channel Download Failed',
description: message,
time: 5000,
});
}
}
return response;
}
export async function updateChannelDownload(channelId, downloadId, url, frequency, title_include, title_exclude, tag_names) {
const body = {
url: url,
frequency: frequency,
settings: {
title_include: title_include,
title_exclude: title_exclude,
tag_names: tag_names,
},
}
url = `${VIDEOS_API}/channels/${channelId}/download/${downloadId}`;
const response = await apiPut(url, body);
if (!response.ok) {
const json = await response.json();
if (json['code'] === 'INVALID_DOWNLOAD') {
toast({
type: 'error',
title: 'Cannot Download!',
description: 'This channel does not have a download record. Modify the frequency.',
time: 5000,
});
} else {
const message = await getErrorMessage(response, 'Could not update Channel download.');
toast({
type: 'error',
title: 'Updating Channel Download Failed',
description: message,
time: 5000,
});
}
}
return response;
}
export async function refreshChannel(channelId) {
let url = `${VIDEOS_API}/channels/refresh/${channelId}`;
const response = await apiPost(url);
@ -744,7 +676,10 @@ export async function getArchive(archiveId) {
const response = await apiGet(`${ARCHIVES_API}/${archiveId}`);
if (response.ok) {
const data = await response.json();
return [data['file_group'], data['history']];
return {
file_group: data['file_group'],
history: data['history'],
}
} else {
const message = await getErrorMessage(response, 'Unable to get Archive. See server logs.');
toast({
@ -764,7 +699,7 @@ export async function postDownload(downloadData) {
throw new Error('downloader is required, but was not provided');
}
let body = {
const body = {
urls: downloadData.urls,
destination: downloadData.destination,
downloader: downloadData.downloader,

View file

@ -1865,7 +1865,7 @@ export function mergeDeep(target, source) {
if (Array.isArray(source[key])) {
// source overwrites target if it is a list with values.
result[key] = source[key] || target[key];
} else if (typeof source[key] === 'object' && typeof target[key] === 'object') {
} else if (typeof source[key] === 'object' && typeof target[key] === 'object' && source[key] !== null) {
result[key] = mergeDeep(target[key] || {}, source[key]);
} else {
result[key] = source[key] !== undefined ? source[key] : target[key];

View file

@ -1,5 +1,5 @@
import React from 'react';
import {ArchiveDownloadForm, ChannelDownloadForm} from './Download';
import {ArchiveDownloadForm, ChannelDownloadForm, EditRSSDownloadForm} from './Download';
import 'semantic-ui-offline/semantic.min.css';
describe('<ArchiveDownloadForm />', () => {
@ -66,3 +66,87 @@ describe('<ChannelDownloadForm/>', () => {
.get('.item').contains('180 Days').click();
});
});
const downloadersResponse = {
"downloaders": [{"name": "archive", "pretty_name": "Archive"}, {
"name": "video",
"pretty_name": "Videos"
}], "manager_disabled": true
};
describe('<EditRSSDownloadForm />', () => {
beforeEach(() => {
const download = {
id: 1,
destination: "destination directory",
downloader: "rss",
frequency: 604800,
next_download: "2024-11-24T16:34:21.014400+00:00",
settings: {
title_include: "included",
title_exclude: "excluded",
video_format: "mp4",
video_resolutions: [
"720p",
"480p",
"maximum"
]
},
sub_downloader: "video",
tag_names: ['Automotive'],
url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC4t8bw1besFTyjW7ZBCOIrw"
};
cy.mountWithTags(<EditRSSDownloadForm download={download}/>);
cy.intercept('GET', '/api/downloaders', downloadersResponse).as('downloaders');
cy.intercept('POST', '**/api/files/search_directories', {
statusCode: 200,
body: {
is_dir: true, channel_directories: [], domain_directories: [],
directories: [{path: "some directory", name: "some directory"}],
},
}).as('searchDirectories');
});
it('can edit videos download', () => {
// Weekly frequency.
cy.get('#url_input').should('have.value',
'https://www.youtube.com/feeds/videos.xml?channel_id=UC4t8bw1besFTyjW7ZBCOIrw');
cy.get('div.divider.text').contains('Weekly');
cy.get('input[name="title_include"]').should('have.value', 'included');
cy.get('input[name="title_exclude"]').should('have.value', 'excluded');
cy.get('#destination_search_form').should('be.visible');
cy.get('div.ui.large.label').contains('Automotive').should('be.visible');
});
it('can submit videos download', () => {
cy.get('button[type="submit"]').should('be.visible');
cy.intercept('PUT', '/api/download/1', (req) => {
expect(req.body).to.deep.equal(JSON.stringify({
"destination": "destination directory",
"downloader": "rss",
"frequency": 604800,
"settings": {
"excluded_urls": null,
"title_exclude": "excluded",
"title_include": "included",
"video_resolutions": [
"720p",
"480p",
"maximum"
],
"video_format": "mp4",
"minimum_duration": null,
"maximum_duration": null
},
"sub_downloader": "video",
"tag_names": ["Automotive"],
"urls": ["https://www.youtube.com/feeds/videos.xml?channel_id=UC4t8bw1besFTyjW7ZBCOIrw"]
}));
req.reply({statusCode: 204});
}).as('downloadPut');
cy.get('button[type="submit"]').click();
});
});

View file

@ -59,10 +59,11 @@ export function DestinationForm({
form,
helpContent,
label = 'Destination',
name = 'destination',
path = 'destination',
required = false,
}) {
const [inputProps, inputAttrs] = form.getCustomProps({name: 'destination', path, required});
const [inputProps, inputAttrs] = form.getCustomProps({name, path, required});
const {disabled, value, onChange} = inputProps;
const helpPopup = helpContent ? <HelpPopup content={helpContent}/> : null;
return <SForm.Field>
@ -72,6 +73,7 @@ export function DestinationForm({
value={value}
onSelect={onChange}
disabled={disabled}
id='destination_search_form'
/>
</SForm.Field>
}
@ -129,6 +131,7 @@ export function DownloaderSelector({form, name = 'sub_downloader', path = 'sub_d
label='Downloader'
placeholder='Select a downloader'
options={downloaders}
id='downloader_selector'
{...inputProps}
/>
}
@ -339,6 +342,8 @@ export function ChannelTagNameForm({form}) {
}
export function VideosDownloadForm({singleDownload = true, onCancel}) {
const [showMessage, setShowMessage] = React.useState(false);
// Keep video settings in session to help user start downloads consistently.
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
@ -366,10 +371,15 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
}
const onSuccess = () => {
setShowMessage(true);
form.reset();
}
const form = useForm({submitter, defaultFormData, onSuccess});
const form = useForm({
submitter,
defaultFormData,
onSuccess,
});
React.useEffect(() => {
const {video_resolutions, video_format} = form.formData.settings;
@ -432,6 +442,7 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
<ChannelTagNameForm form={form}/>
</Grid.Column>
</Grid.Row>
{showMessage && <SuccessfulDownloadSubmitMessage/>}
<Grid.Row>
<Grid.Column>
<DownloadFormButtons onCancel={localOnCancel} form={form}/>
@ -476,6 +487,7 @@ export function ChannelDownloadForm({
clearOnSuccess = true,
channel_id = null,
}) {
const [showMessage, setShowMessage] = React.useState(false);
// May have received submitter from EditChannelDownloadForm.
submitter = submitter || (async (formData) => {
@ -491,6 +503,13 @@ export function ChannelDownloadForm({
await postDownload(downloadData);
});
const localOnSuccess = async () => {
setShowMessage(true);
if (onSuccess) {
onSuccess();
}
}
// Keep video settings in session to help user start downloads consistently.
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
@ -520,7 +539,7 @@ export function ChannelDownloadForm({
submitter,
defaultFormData: mergeDeep(emptyFormData, download),
emptyFormData,
onSuccess,
onSuccess: localOnSuccess,
clearOnSuccess,
});
@ -608,6 +627,7 @@ export function ChannelDownloadForm({
<ChannelTagNameForm form={form}/>
</Grid.Column>
</Grid.Row>
{showMessage && <SuccessfulDownloadSubmitMessage/>}
<Grid.Row>
<Grid.Column textAlign='right'>
{actionsElm}
@ -632,7 +652,7 @@ export function EditChannelDownloadForm({
downloader: formData.downloader,
frequency: formData.frequency,
settings: formData.settings,
tag_names: formData.tag_names,
tag_names: formData.tag_names || [],
urls: [formData.url,],
}
await putDownload(download.id, downloadData);
@ -651,12 +671,15 @@ export function EditChannelDownloadForm({
}
function SuccessfulDownloadSubmitMessage() {
return <Message positive>
<Message.Header>Download Submitted</Message.Header>
<Message.Content>
<Link to='/admin'><Icon name='checkmark'/> View downloads</Link>
</Message.Content>
</Message>
return <Grid.Row>
<Grid.Column><Message positive>
<Message.Header>Download Submitted</Message.Header>
<Message.Content>
<Link to='/admin'><Icon name='checkmark'/> View downloads</Link>
</Message.Content>
</Message>
</Grid.Column>
</Grid.Row>
}
export function ArchiveDownloadForm({download, onCancel}) {
@ -700,12 +723,7 @@ export function ArchiveDownloadForm({download, onCancel}) {
<DownloadTagsSelector form={form}/>
</Grid.Column>
</Grid.Row>
{showMessage &&
<Grid.Row>
<Grid.Column>
<SuccessfulDownloadSubmitMessage/>
</Grid.Column>
</Grid.Row>}
{showMessage && <SuccessfulDownloadSubmitMessage/>}
<Grid.Row>
<Grid.Column textAlign='right'>
<DownloadFormButtons onCancel={onCancel} form={form}/>
@ -728,7 +746,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
frequency: formData.frequency,
settings: formData.settings,
sub_downloader: formData.sub_downloader,
tag_names: formData.tag_names,
tag_names: formData.tag_names || [],
urls: [formData.url],
}
await postDownload(downloadData);
@ -840,12 +858,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
</Grid.Column>
</Grid.Row>
{downloaderRows}
{showMessage &&
<Grid.Row>
<Grid.Column>
<SuccessfulDownloadSubmitMessage/>
</Grid.Column>
</Grid.Row>}
{showMessage && <SuccessfulDownloadSubmitMessage/>}
<Grid.Row>
<Grid.Column textAlign='right'>
{actionsElm}
@ -859,11 +872,12 @@ export function EditRSSDownloadForm({download, onDelete, onCancel, actions = Edi
const submitter = async (formData) => {
const downloadData = {
destination: formData.destination,
downloader: formData.downloader,
frequency: formData.frequency,
settings: formData.settings,
sub_downloader: formData.sub_downloader,
tag_names: formData.tag_names,
tag_names: formData.tag_names || [],
urls: [formData.url,],
}
await putDownload(download.id, downloadData);
@ -927,12 +941,7 @@ export function EditZimDownloadForm({download, onDelete, onCancel, actions = Edi
<DownloadFrequencySelector form={form} longFrequenciesAvailable={true}/>
</Grid.Column>
</Grid.Row>
{showMessage &&
<Grid.Row>
<Grid.Column>
<SuccessfulDownloadSubmitMessage/>
</Grid.Column>
</Grid.Row>}
{showMessage && <SuccessfulDownloadSubmitMessage/>}
<Grid.Row>
<Grid.Column textAlign='right'>
{actionsElm}
@ -997,12 +1006,7 @@ export function FilesDownloadForm({
<DestinationForm required form={form}/>
</Grid.Column>
</Grid.Row>
{showMessage &&
<Grid.Row>
<Grid.Column>
<SuccessfulDownloadSubmitMessage/>
</Grid.Column>
</Grid.Row>}
{showMessage && <SuccessfulDownloadSubmitMessage/>}
<Grid.Row>
<Grid.Column textAlign='right'>
{actionsElm}
@ -1046,8 +1050,11 @@ export function ScrapeFilesDownloadForm({
download,
submitter,
clearOnSuccess,
onDelete,
onCancel,
onSuccess,
actions = DownloadFormButtons,
singleDownload = false,
}) {
const [showMessage, setShowMessage] = React.useState(false);
@ -1055,7 +1062,7 @@ export function ScrapeFilesDownloadForm({
const downloadData = {
downloader: Downloaders.ScrapeHtml,
sub_downloader: Downloaders.File,
tag_names: formData.tag_names,
tag_names: formData.tag_names || [],
destination: formData.destination,
urls: formData.urls.split(/\r?\n/),
settings: formData.settings,
@ -1074,15 +1081,28 @@ export function ScrapeFilesDownloadForm({
}
};
const localOnSuccess = async () => {
setShowMessage(true);
if (onSuccess) {
onSuccess();
}
}
const form = useForm({
submitter,
defaultFormData: mergeDeep(emptyFormData, download),
emptyFormData,
clearOnSuccess,
onSuccess: async () => setShowMessage(true),
onSuccess: localOnSuccess,
});
const actionsElm = actions({onCancel, form});
const urlInput = singleDownload ?
<UrlInput required form={form} path='url'/>
: <UrlsTextarea required form={form}/>;
// Default to "new" download buttons.
actions = actions || DownloadFormButtons;
const actionsElm = actions({onDelete, onCancel, form});
return <Form>
<Header as='h3'><Icon name='file alternate' color='red'/> Scrape Files</Header>
@ -1091,7 +1111,7 @@ export function ScrapeFilesDownloadForm({
<Grid>
<Grid.Row>
<Grid.Column>
<UrlsTextarea form={form}/>
{urlInput}
</Grid.Column>
</Grid.Row>
<Grid.Row columns={2}>
@ -1110,12 +1130,7 @@ export function ScrapeFilesDownloadForm({
<MaximumPagesInputForm form={form} required={true}/>
</Grid.Column>
</Grid.Row>
{showMessage &&
<Grid.Row>
<Grid.Column>
<SuccessfulDownloadSubmitMessage/>
</Grid.Column>
</Grid.Row>}
{showMessage && <SuccessfulDownloadSubmitMessage/>}
<Grid.Row>
<Grid.Column>
{actionsElm}
@ -1125,6 +1140,30 @@ export function ScrapeFilesDownloadForm({
</Form>
}
export function EditScrapeFilesDownloadForm({download, onDelete, onCancel, onSuccess}) {
const submitter = async (formData) => {
const downloadData = {
downloader: Downloaders.ScrapeHtml,
sub_downloader: Downloaders.File,
tag_names: formData.tag_names || [],
destination: formData.destination,
urls: [formData.url],
settings: formData.settings,
}
await postDownload(downloadData);
};
return <ScrapeFilesDownloadForm
download={download}
submitter={submitter}
onDelete={onDelete}
onCancel={onCancel}
onSuccess={onSuccess}
actions={EditDownloadFormButtons}
singleDownload={true}
/>
}
export function DownloadMenu({onOpen, disabled}) {
const [downloader, setDownloader] = useState();

View file

@ -31,7 +31,12 @@ import {
} from "semantic-ui-react";
import {Button, Header, Modal, ModalActions, ModalContent, ModalHeader, Placeholder, Segment, Table} from "../Theme";
import {useDownloads} from "../../hooks/customHooks";
import {EditChannelDownloadForm, EditRSSDownloadForm, EditZimDownloadForm} from "../Download";
import {
EditChannelDownloadForm,
EditRSSDownloadForm,
EditScrapeFilesDownloadForm,
EditZimDownloadForm
} from "../Download";
import {Downloaders} from "../Vars";
function ClearDownloadsButton({callback}) {
@ -200,6 +205,13 @@ function RecurringDownloadRow({download, fetchDownloads, onDelete}) {
onCancel={handleEditClose}
onSuccess={onSuccess}
/>;
} else if (downloader === Downloaders.ScrapeHtml) {
editForm = <EditScrapeFilesDownloadForm
download={download}
onDelete={localOnDelete}
onCancel={handleEditClose}
onSuccess={onSuccess}
/>;
}
const editModal = <Modal closeIcon

View file

@ -215,7 +215,7 @@ export const useArchive = (archiveId) => {
const fetchArchive = async () => {
try {
const [file_group, history] = await getArchive(archiveId);
const {file_group, history} = await getArchive(archiveId);
setArchiveFileGroup(file_group);
setHistory(history);
} catch (e) {

View file

@ -107,34 +107,6 @@ def channel_refresh(_: Request, channel_id: int):
return response.empty()
@channel_bp.post('/<channel_id:int>/download', name='post_channel_download')
@channel_bp.put('/<channel_id:int>/download/<download_id:int>', name='put_channel_download')
@openapi.description('Create or update a Channel Download record')
@validate(schema.ChannelDownloadRequest)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@wrol_mode_check
async def channel_download(_: Request, channel_id: int, body: schema.ChannelDownloadRequest, download_id: int = None):
if download_id:
await lib.update_channel_download(
channel_id,
download_id,
body.url,
body.frequency,
body.settings,
)
else:
await lib.create_channel_download(
channel_id,
body.url,
body.frequency,
body.settings,
)
if download_manager.is_disabled:
# Warn the user that downloads are disabled.
Events.send_downloads_disabled('Channel download created. But downloads are disabled.')
return response.empty()
@channel_bp.post('/<channel_id:int>/tag')
@openapi.definition(
description='Tag/untag a Channel with a single tag',

View file

@ -1,4 +1,3 @@
import asyncio
import json
import pathlib
import shutil
@ -10,6 +9,7 @@ import mock
import pytest
from modules.videos.common import get_videos_directory
from modules.videos.conftest import simple_channel
from modules.videos.downloader import ChannelDownloader
from modules.videos.lib import save_channels_config, get_channels_config
from modules.videos.models import Channel, Video
@ -71,7 +71,7 @@ def test_channel_no_download_frequency(test_client, test_session, test_directory
# Update the Channel with a frequency.
new_download = {'urls': [simple_channel.url], 'frequency': 10, 'downloader': ChannelDownloader.name}
request, response = test_client.post('/api/download', json=new_download)
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
assert response.status_code == HTTPStatus.CREATED, response.json
# Download is scheduled. Download is related to Channel.
assert len(download_manager.get_downloads(test_session)) == 1
@ -410,46 +410,47 @@ async def test_get_channel_videos_pagination(test_session, simple_channel, video
@pytest.mark.asyncio
async def test_create_channel_download_api(async_client, test_session, simple_channel, tag_factory):
tag = await tag_factory()
async def test_channel_download_id(async_client, test_session, tag_factory, simple_channel, test_download_manager,
test_downloader):
tag = await tag_factory()
# Create Download.
body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
request, response = await async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download: Download = test_session.query(Download).one()
assert download.url == 'https://example.com/1' and not download.settings.get('tag_names')
assert download.settings.get('title_include') is None
# A recurring Download can be displayed on a Channel's page.
body = dict(
urls=['https://example.com/channel1'],
tag_names=[tag.name],
downloader=test_downloader.name,
settings=dict(channel_id=simple_channel.id),
frequency=120,
)
request, response = await async_client.post(f'/api/download', json=body)
assert response.status_code == HTTPStatus.CREATED
download = test_session.query(Download).one()
assert download.url == 'https://example.com/channel1'
assert download.channel_id == simple_channel.id
# Create another Download. Previous Downloads is untouched.
body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}}
request, response = await async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download1, download2 = test_session.query(Download).order_by(Download.url).all()
assert download1.url == 'https://example.com/1' and not download1.settings.get('tag_names')
assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ]
test_session.flush(simple_channel)
assert len(simple_channel.downloads) == 2
# Change frequency of first Download.
body = {'url': 'https://example.com/1', 'frequency': 123, 'settings': {'title_include': 'foo,bar'}}
request, response = await async_client.put(
f'/api/videos/channels/{simple_channel.id}/download/{download1.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download1, download2 = test_session.query(Download).order_by(Download.url).all()
assert download1.url == 'https://example.com/1'
assert download1.settings == {'title_include': 'foo,bar'}
assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ]
test_session.flush(simple_channel)
assert len(simple_channel.downloads) == 2
# Deleting Download does not delete Channel.
request, response = await async_client.delete(f'/api/download/{download1.id}')
# A Channel relationship can be removed from a Download.
body = dict(
urls=['https://example.com/channel1'],
tag_names=[tag.name],
downloader=test_downloader.name,
settings=dict(),
frequency=240,
)
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 1
assert test_session.query(Channel).count() == 1
download = test_session.query(Download).one()
assert download.channel_id is None
assert download.frequency == 240
# A once-Download cannot be associated with a Channel.
body = dict(
urls=['https://example.com/channel1'],
tag_names=[tag.name],
downloader=test_downloader.name,
settings=dict(channel_id=simple_channel.id),
)
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio

View file

@ -151,7 +151,7 @@ def test_api_download(test_session, test_client, test_directory):
content = dict(urls=['https://example.com/video1', ], downloader='video', destination='dest',
settings=dict(excluded_urls='example.com'))
request, response = test_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert response.status_code == HTTPStatus.CREATED
download = test_session.query(Download).one()
assert download.url == 'https://example.com/video1'
@ -167,15 +167,14 @@ async def test_api_download_video_tags(test_session, async_client, tag_factory):
tag1 = await tag_factory()
tag2 = await tag_factory()
content = dict(urls=['https://example.com/video1', ], downloader='video',
settings={'tag_names': [tag1.name, tag2.name]})
content = dict(urls=['https://example.com/video1', ], downloader='video', tag_names=[tag1.name, tag2.name])
request, response = await async_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert response.status_code == HTTPStatus.CREATED
download = test_session.query(Download).one()
assert download.url == 'https://example.com/video1'
assert download.downloader == 'video'
assert download.settings['tag_names'] == [tag1.name, tag2.name]
assert download.tag_names == [tag1.name, tag2.name]
@pytest.mark.asyncio

View file

@ -453,7 +453,7 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac
@pytest.mark.asyncio
async def test_channel_download_crud(test_session, async_client, assert_downloads, tag_factory):
"""Test creatin"""
"""Test creating more complex Channel Download."""
one, two = await tag_factory(), await tag_factory()
download = {
'urls': ['https://www.youtube.com/@wrolpi/videos'],
@ -476,10 +476,11 @@ async def test_channel_download_crud(test_session, async_client, assert_download
}
request, response = await async_client.post('/api/download', json=download)
assert response.status == HTTPStatus.NO_CONTENT
assert response.status == HTTPStatus.CREATED, request.body
download = test_session.query(Download).one()
assert download.url == 'https://www.youtube.com/@wrolpi/videos'
assert download.destination is None
assert download.tag_names == ['one', 'two']
assert download.downloader == 'video_channel'
assert download.sub_downloader == 'video'

View file

@ -5,7 +5,7 @@ from typing import Optional, List, Dict
@dataclass
class ZimSearchRequest:
search_str: Optional[str] = None
tag_names: List[str] = field(default_factory=lambda: list())
tag_names: List[str] = field(default_factory=list)
offset: Optional[int] = 0
limit: Optional[int] = 10
@ -98,5 +98,5 @@ class OutdatedZims:
@dataclass
class SearchEstimateRequest:
search_str: Optional[str] = None
tag_names: List[str] = field(default_factory=lambda: list())
tag_names: List[str] = field(default_factory=list)
any_tag: bool = False

View file

@ -584,6 +584,10 @@ class DownloadManager:
if not frequency or not isinstance(frequency, int):
raise ValueError('Recurring download must have a frequency!')
from wrolpi.scrape_downloader import ScrapeHTMLDownloader
if downloader_name == ScrapeHTMLDownloader.name and frequency:
raise InvalidDownload(f'Cannot schedule recurring download for {downloader_name=}')
download, = self.create_downloads([url, ], session=session, downloader_name=downloader_name,
reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
destination=destination, tag_names=tag_names, settings=settings)
@ -601,17 +605,22 @@ class DownloadManager:
@optional_session
def update_download(self, id_: int, url: str, downloader: str,
destination: str | pathlib.Path = None, tag_names: List[str] = None,
sub_downloader: str | None = None,
sub_downloader: str | None = None, frequency: int = None,
settings: Dict = None, session: Session = None) -> Download:
download = self.find_download(session=session, id_=id_)
if settings and settings.get('channel_id') and not frequency:
raise InvalidDownload(f'A once-download cannot be associated with a Channel')
download.url = url
download.downloader = downloader
download.frequency = frequency
# Preserve existing settings, unless new settings are provided.
download.settings = settings if settings is not None else download.settings
# Use provided params even if empty.
download.destination = destination or None
download.tag_names = tag_names or None
download.sub_downloader = sub_downloader or None
# Remove Channel relationship, if necessary.
download.channel_id = (settings or dict()).get('channel_id')
return download
@wrol_mode_check

View file

@ -1,7 +1,10 @@
import asyncio
from http import HTTPStatus
import mock
import pytest
from wrolpi.dates import Seconds
from wrolpi.downloader import DownloadResult
from wrolpi.files.downloader import FileDownloader
from wrolpi.scrape_downloader import ScrapeHTMLDownloader
@ -76,3 +79,18 @@ async def test_scrape_html_downloader_html(test_directory, test_session, test_do
'https://example.com/dir',
'https://example.com/dir/other.html',
])
@pytest.mark.asyncio
async def test_scrape_html_downloader_api(async_client, test_session, test_download_manager):
"""Scrape Download cannot be recurring."""
body = dict(
urls=['https://example.com'],
destination='uploads',
downloader='scrape_html',
sub_downloader='file',
frequency=Seconds.week,
tag_names=[],
settings=dict(depth=1, max_pages=1, suffix='.mp4'))
request, response = await async_client.post('/api/download', json=body)
assert response.status == HTTPStatus.BAD_REQUEST

View file

@ -255,13 +255,15 @@ def valid_regex(_: Request, body: schema.RegexRequest):
@api_bp.post('/download')
@openapi.description('Download all the URLs that are provided.')
@validate(schema.DownloadRequest)
@openapi.definition(
description='Download all the URLs that are provided.',
body=schema.DownloadRequest,
validate=True,
)
@wrol_mode_check
async def post_download(_: Request, body: schema.DownloadRequest):
downloader = download_manager.find_downloader_by_name(body.downloader)
if not downloader:
raise InvalidDownload(f'Cannot find downloader with name {body.downloader}')
# Raises an InvalidDownload if the Downloader cannot be found.
download_manager.find_downloader_by_name(body.downloader)
kwargs = dict(downloader_name=body.downloader,
sub_downloader_name=body.sub_downloader, reset_attempts=True,
@ -275,17 +277,20 @@ async def post_download(_: Request, body: schema.DownloadRequest):
# Downloads are disabled, warn the user.
Events.send_downloads_disabled('Download created. But, downloads are disabled.')
return response.empty()
return response.empty(status=HTTPStatus.CREATED)
@api_bp.put('/download/<download_id:int>')
@openapi.description('Update properties of the Download')
@validate(schema.DownloadRequest)
@openapi.definition(
description='Update properties of the Download',
body=schema.DownloadRequest,
validate=True,
)
@wrol_mode_check
async def put_download(_: Request, download_id: int, body: schema.DownloadRequest):
downloader = download_manager.find_downloader_by_name(body.downloader)
if not downloader:
raise ValidationError(f'Cannot find downloader with name {body.downloader}')
# Raises an InvalidDownload if the Downloader cannot be found.
download_manager.find_downloader_by_name(body.downloader)
if len(body.urls) != 1:
raise ValidationError('Only one URL can be specified when updating a Download')
@ -295,6 +300,7 @@ async def put_download(_: Request, download_id: int, body: schema.DownloadReques
url=body.urls[0],
downloader=body.downloader,
destination=body.destination,
frequency=body.frequency,
tag_names=body.tag_names,
sub_downloader=body.sub_downloader,
settings=body.settings,

View file

@ -80,8 +80,8 @@ class EchoResponse:
@dataclass
class DownloadSettings:
channel_id: int = None
channel_tag_name: List[str] = field(default_factory=lambda: list())
channel_id: Optional[int] = None
channel_tag_name: List[str] = field(default_factory=list)
depth: Optional[int] = None
download_metadata_only: bool = False
download_order: Optional[str] = None
@ -90,16 +90,13 @@ class DownloadSettings:
maximum_duration: Optional[int] = None
minimum_duration: Optional[int] = None
suffix: Optional[str] = None
tag_names: List[str] = field(default_factory=lambda: list())
title_exclude: str = None
title_include: str = None
title_exclude: Optional[str] = None
title_include: Optional[str] = None
video_count_limit: Optional[int] = None
video_format: str = None
video_resolutions: List[str] = field(default_factory=lambda: list())
video_format: Optional[str] = None
video_resolutions: List[str] = field(default_factory=list)
def __post_init__(self):
self.tag_names = [i.strip() for i in self.tag_names] if self.tag_names else []
if self.excluded_urls and self.excluded_urls.endswith(','):
raise ValidationError('Excluded urls cannot end with ,')
if self.excluded_urls and self.excluded_urls.startswith(','):
@ -128,7 +125,7 @@ class DownloadRequest:
urls: List[str]
downloader: str
destination: Optional[str] = None
tag_names: List[str] = field(default_factory=list)
tag_names: Optional[List[str]] = field(default_factory=list)
frequency: Optional[int] = None
sub_downloader: Optional[str] = None
settings: Optional[dict] = field(default_factory=dict)
@ -146,6 +143,8 @@ class DownloadRequest:
destination = get_relative_to_media_directory(destination)
self.destination = str(destination)
self.tag_names = self.tag_names or []
# Validate settings contents. Remove empty values.
settings = self.settings or dict()
self.settings = {k: v for k, v in DownloadSettings(**settings).__dict__.items() if v not in ([], None)}
@ -226,8 +225,8 @@ class SearchSuggestionsRequest:
@dataclass
class SearchFileEstimateRequest:
search_str: Optional[str] = None
mimetypes: List[str] = field(default_factory=lambda: list())
tag_names: List[str] = field(default_factory=lambda: list())
mimetypes: List[str] = field(default_factory=list)
tag_names: List[str] = field(default_factory=list)
months: Optional[List[int]] = None
from_year: Optional[int] = None
to_year: Optional[int] = None
@ -240,7 +239,7 @@ class SearchFileEstimateRequest:
@dataclass
class SearchOtherEstimateRequest:
tag_names: List[str] = field(default_factory=lambda: list())
tag_names: List[str] = field(default_factory=list)
@dataclass

View file

@ -382,7 +382,7 @@ async def test_crud_download(async_client, test_session, test_download_manager,
settings=dict(excluded_urls='example.org,something'),
)
request, response = await async_client.post('/api/download', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT, response.body
assert response.status_code == HTTPStatus.CREATED, response.body
download: Download = test_session.query(Download).one()
assert download.id

View file

@ -430,42 +430,46 @@ async def test_download_crud(test_session, async_client, test_download_manager_c
with mock.patch('wrolpi.downloader.DownloadManager.dispatch_downloads', dispatch_downloads):
# Create a single recurring Download.
content = dict(
body = dict(
urls=['https://example.com/1', ],
frequency=1_000,
downloader='archive',
tag_names=None, # tag_names can be a None.
)
request, response = await async_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = await async_client.post('/api/download', content=json.dumps(body))
assert response.status_code == HTTPStatus.CREATED
assert {i.url for i in test_session.query(Download)} == {'https://example.com/1', }
# Create two once-downloads.
content = dict(
body = dict(
urls=['https://example.com/2', 'https://example.com/3'],
downloader='archive',
# tag_names=None, # tag_names can be missing.
)
request, response = await async_client.post('/api/download', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = await async_client.post('/api/download', json=body)
assert response.status_code == HTTPStatus.CREATED
assert {i.url for i in test_session.query(Download)} == {f'https://example.com/{i}' for i in range(1, 4)}
download1, download2, download3 = test_session.query(Download).order_by(Download.url).all()
assert not download1.settings
expected_download1 = dict(download1.__json__()).copy()
expected_download = dict(download1.__json__()).copy()
# Update a download.
content = dict(
body = dict(
urls=['https://example.com/1'],
downloader='archive',
settings={'tag_names': [tag1.name, tag2.name]},
frequency=1_000,
tag_names=[tag1.name, tag2.name],
)
expected_download1.update({'url': 'https://example.com/1', 'settings': {'tag_names': [tag1.name, tag2.name]}})
request, response = await async_client.put(f'/api/download/{download1.id}', json=content)
expected_download.update({'url': 'https://example.com/1', 'tag_names': [tag1.name, tag2.name]})
request, response = await async_client.put(f'/api/download/{download1.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
test_session.flush([download2])
download2 = test_session.query(Download).filter_by(id=expected_download1['id']).first()
assert download2.__json__() == expected_download1
download2 = test_session.query(Download).filter_by(id=expected_download['id']).first()
assert download2.__json__() == expected_download
assert download2.tag_names == [tag1.name, tag2.name]
def test_get_downloaders(test_client):
@ -514,33 +518,6 @@ async def test_restart_download(test_session, async_client, test_download_manage
assert download.is_deferred, download.status
@pytest.mark.asyncio
async def test_put_download(test_session, async_client, test_download_manager, test_downloader, channel_factory):
"""A Download can be updated using a PUT request."""
channel = channel_factory()
download = test_download_manager.create_download('https://example.com', test_downloader.name)
assert not download.settings
body = dict(
downloader=test_downloader.name,
urls=['https://example.com/new-url'], # URL can be changed.
settings=dict(channel_id=channel.id),
)
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 1, 'Download should have been updated'
download = test_session.query(Download).one()
assert download.url == 'https://example.com/new-url', 'Download URL was not changed'
assert download.settings == dict(channel_id=channel.id), 'Download settings were not changed'
# Settings can be cleared by sending `None`
body['settings'] = None
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
assert download.settings == dict(), 'Download settings were not changed'
def test_get_global_statistics(test_session, test_client):
request, response = test_client.get('/api/statistics')
assert response.json['global_statistics']['db_size'] > 1