diff --git a/app/src/api.js b/app/src/api.js
index 743bc8a1..6f7c5fcc 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -413,74 +413,6 @@ export async function getVideosStatistics() {
return response
}
-export async function createChannelDownload(channelId, url, frequency, title_include, title_exclude, tag_names) {
- const body = {
- url: url,
- frequency: frequency,
- settings: {
- title_include: title_include,
- title_exclude: title_exclude,
- tag_names: tag_names,
- },
- }
- url = `${VIDEOS_API}/channels/${channelId}/download`;
- const response = await apiPost(url, body);
- if (!response.ok) {
- const json = await response.json();
- if (json['code'] === 'INVALID_DOWNLOAD') {
- toast({
- type: 'error',
- title: 'Cannot Download!',
- description: 'This channel does not have a download record. Modify the frequency.',
- time: 5000,
- });
- } else {
- const message = await getErrorMessage(response, 'Could not create Channel download.');
- toast({
- type: 'error',
- title: 'Creating Channel Download Failed',
- description: message,
- time: 5000,
- });
- }
- }
- return response;
-}
-
-export async function updateChannelDownload(channelId, downloadId, url, frequency, title_include, title_exclude, tag_names) {
- const body = {
- url: url,
- frequency: frequency,
- settings: {
- title_include: title_include,
- title_exclude: title_exclude,
- tag_names: tag_names,
- },
- }
- url = `${VIDEOS_API}/channels/${channelId}/download/${downloadId}`;
- const response = await apiPut(url, body);
- if (!response.ok) {
- const json = await response.json();
- if (json['code'] === 'INVALID_DOWNLOAD') {
- toast({
- type: 'error',
- title: 'Cannot Download!',
- description: 'This channel does not have a download record. Modify the frequency.',
- time: 5000,
- });
- } else {
- const message = await getErrorMessage(response, 'Could not update Channel download.');
- toast({
- type: 'error',
- title: 'Updating Channel Download Failed',
- description: message,
- time: 5000,
- });
- }
- }
- return response;
-}
-
export async function refreshChannel(channelId) {
let url = `${VIDEOS_API}/channels/refresh/${channelId}`;
const response = await apiPost(url);
@@ -744,7 +676,10 @@ export async function getArchive(archiveId) {
const response = await apiGet(`${ARCHIVES_API}/${archiveId}`);
if (response.ok) {
const data = await response.json();
- return [data['file_group'], data['history']];
+ return {
+ file_group: data['file_group'],
+ history: data['history'],
+ }
} else {
const message = await getErrorMessage(response, 'Unable to get Archive. See server logs.');
toast({
@@ -764,7 +699,7 @@ export async function postDownload(downloadData) {
throw new Error('downloader is required, but was not provided');
}
- let body = {
+ const body = {
urls: downloadData.urls,
destination: downloadData.destination,
downloader: downloadData.downloader,
diff --git a/app/src/components/Common.js b/app/src/components/Common.js
index 60780617..8417a730 100644
--- a/app/src/components/Common.js
+++ b/app/src/components/Common.js
@@ -1865,7 +1865,7 @@ export function mergeDeep(target, source) {
if (Array.isArray(source[key])) {
// source overwrites target if it is a list with values.
result[key] = source[key] || target[key];
- } else if (typeof source[key] === 'object' && typeof target[key] === 'object') {
+ } else if (typeof source[key] === 'object' && typeof target[key] === 'object' && source[key] !== null) {
result[key] = mergeDeep(target[key] || {}, source[key]);
} else {
result[key] = source[key] !== undefined ? source[key] : target[key];
diff --git a/app/src/components/Download.cy.js b/app/src/components/Download.cy.js
index 08868d33..39195b35 100644
--- a/app/src/components/Download.cy.js
+++ b/app/src/components/Download.cy.js
@@ -1,5 +1,5 @@
import React from 'react';
-import {ArchiveDownloadForm, ChannelDownloadForm} from './Download';
+import {ArchiveDownloadForm, ChannelDownloadForm, EditRSSDownloadForm} from './Download';
import 'semantic-ui-offline/semantic.min.css';
describe('', () => {
@@ -66,3 +66,87 @@ describe('', () => {
.get('.item').contains('180 Days').click();
});
});
+
+const downloadersResponse = {
+ "downloaders": [{"name": "archive", "pretty_name": "Archive"}, {
+ "name": "video",
+ "pretty_name": "Videos"
+ }], "manager_disabled": true
+};
+
+describe('', () => {
+ beforeEach(() => {
+ const download = {
+ id: 1,
+ destination: "destination directory",
+ downloader: "rss",
+ frequency: 604800,
+ next_download: "2024-11-24T16:34:21.014400+00:00",
+ settings: {
+ title_include: "included",
+ title_exclude: "excluded",
+ video_format: "mp4",
+ video_resolutions: [
+ "720p",
+ "480p",
+ "maximum"
+ ]
+ },
+ sub_downloader: "video",
+ tag_names: ['Automotive'],
+ url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC4t8bw1besFTyjW7ZBCOIrw"
+ };
+ cy.mountWithTags();
+ cy.intercept('GET', '/api/downloaders', downloadersResponse).as('downloaders');
+ cy.intercept('POST', '**/api/files/search_directories', {
+ statusCode: 200,
+ body: {
+ is_dir: true, channel_directories: [], domain_directories: [],
+ directories: [{path: "some directory", name: "some directory"}],
+ },
+ }).as('searchDirectories');
+
+ });
+
+ it('can edit videos download', () => {
+ // Weekly frequency.
+ cy.get('#url_input').should('have.value',
+ 'https://www.youtube.com/feeds/videos.xml?channel_id=UC4t8bw1besFTyjW7ZBCOIrw');
+ cy.get('div.divider.text').contains('Weekly');
+ cy.get('input[name="title_include"]').should('have.value', 'included');
+ cy.get('input[name="title_exclude"]').should('have.value', 'excluded');
+ cy.get('#destination_search_form').should('be.visible');
+ cy.get('div.ui.large.label').contains('Automotive').should('be.visible');
+ });
+
+ it('can submit videos download', () => {
+ cy.get('button[type="submit"]').should('be.visible');
+
+ cy.intercept('PUT', '/api/download/1', (req) => {
+ expect(req.body).to.deep.equal(JSON.stringify({
+ "destination": "destination directory",
+ "downloader": "rss",
+ "frequency": 604800,
+ "settings": {
+ "excluded_urls": null,
+ "title_exclude": "excluded",
+ "title_include": "included",
+ "video_resolutions": [
+ "720p",
+ "480p",
+ "maximum"
+ ],
+ "video_format": "mp4",
+ "minimum_duration": null,
+ "maximum_duration": null
+ },
+ "sub_downloader": "video",
+ "tag_names": ["Automotive"],
+ "urls": ["https://www.youtube.com/feeds/videos.xml?channel_id=UC4t8bw1besFTyjW7ZBCOIrw"]
+ }));
+ req.reply({statusCode: 204});
+ }).as('downloadPut');
+
+ cy.get('button[type="submit"]').click();
+ });
+});
diff --git a/app/src/components/Download.js b/app/src/components/Download.js
index 9eb11ae8..e86d77e6 100644
--- a/app/src/components/Download.js
+++ b/app/src/components/Download.js
@@ -59,10 +59,11 @@ export function DestinationForm({
form,
helpContent,
label = 'Destination',
+ name = 'destination',
path = 'destination',
required = false,
}) {
- const [inputProps, inputAttrs] = form.getCustomProps({name: 'destination', path, required});
+ const [inputProps, inputAttrs] = form.getCustomProps({name, path, required});
const {disabled, value, onChange} = inputProps;
const helpPopup = helpContent ? : null;
return
@@ -72,6 +73,7 @@ export function DestinationForm({
value={value}
onSelect={onChange}
disabled={disabled}
+ id='destination_search_form'
/>
}
@@ -129,6 +131,7 @@ export function DownloaderSelector({form, name = 'sub_downloader', path = 'sub_d
label='Downloader'
placeholder='Select a downloader'
options={downloaders}
+ id='downloader_selector'
{...inputProps}
/>
}
@@ -339,6 +342,8 @@ export function ChannelTagNameForm({form}) {
}
export function VideosDownloadForm({singleDownload = true, onCancel}) {
+ const [showMessage, setShowMessage] = React.useState(false);
+
// Keep video settings in session to help user start downloads consistently.
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
@@ -366,10 +371,15 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
}
const onSuccess = () => {
+ setShowMessage(true);
form.reset();
}
- const form = useForm({submitter, defaultFormData, onSuccess});
+ const form = useForm({
+ submitter,
+ defaultFormData,
+ onSuccess,
+ });
React.useEffect(() => {
const {video_resolutions, video_format} = form.formData.settings;
@@ -432,6 +442,7 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
+ {showMessage && }
@@ -476,6 +487,7 @@ export function ChannelDownloadForm({
clearOnSuccess = true,
channel_id = null,
}) {
+ const [showMessage, setShowMessage] = React.useState(false);
// May have received submitter from EditChannelDownloadForm.
submitter = submitter || (async (formData) => {
@@ -491,6 +503,13 @@ export function ChannelDownloadForm({
await postDownload(downloadData);
});
+ const localOnSuccess = async () => {
+ setShowMessage(true);
+ if (onSuccess) {
+ onSuccess();
+ }
+ }
+
// Keep video settings in session to help user start downloads consistently.
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
@@ -520,7 +539,7 @@ export function ChannelDownloadForm({
submitter,
defaultFormData: mergeDeep(emptyFormData, download),
emptyFormData,
- onSuccess,
+ onSuccess: localOnSuccess,
clearOnSuccess,
});
@@ -608,6 +627,7 @@ export function ChannelDownloadForm({
+ {showMessage && }
{actionsElm}
@@ -632,7 +652,7 @@ export function EditChannelDownloadForm({
downloader: formData.downloader,
frequency: formData.frequency,
settings: formData.settings,
- tag_names: formData.tag_names,
+ tag_names: formData.tag_names || [],
urls: [formData.url,],
}
await putDownload(download.id, downloadData);
@@ -651,12 +671,15 @@ export function EditChannelDownloadForm({
}
function SuccessfulDownloadSubmitMessage() {
- return
- Download Submitted
-
- View downloads
-
-
+ return
+
+ Download Submitted
+
+ View downloads
+
+
+
+
}
export function ArchiveDownloadForm({download, onCancel}) {
@@ -700,12 +723,7 @@ export function ArchiveDownloadForm({download, onCancel}) {
- {showMessage &&
-
-
-
-
- }
+ {showMessage && }
@@ -728,7 +746,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
frequency: formData.frequency,
settings: formData.settings,
sub_downloader: formData.sub_downloader,
- tag_names: formData.tag_names,
+ tag_names: formData.tag_names || [],
urls: [formData.url],
}
await postDownload(downloadData);
@@ -840,12 +858,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
{downloaderRows}
- {showMessage &&
-
-
-
-
- }
+ {showMessage && }
{actionsElm}
@@ -859,11 +872,12 @@ export function EditRSSDownloadForm({download, onDelete, onCancel, actions = Edi
const submitter = async (formData) => {
const downloadData = {
+ destination: formData.destination,
downloader: formData.downloader,
frequency: formData.frequency,
settings: formData.settings,
sub_downloader: formData.sub_downloader,
- tag_names: formData.tag_names,
+ tag_names: formData.tag_names || [],
urls: [formData.url,],
}
await putDownload(download.id, downloadData);
@@ -927,12 +941,7 @@ export function EditZimDownloadForm({download, onDelete, onCancel, actions = Edi
- {showMessage &&
-
-
-
-
- }
+ {showMessage && }
{actionsElm}
@@ -997,12 +1006,7 @@ export function FilesDownloadForm({
- {showMessage &&
-
-
-
-
- }
+ {showMessage && }
{actionsElm}
@@ -1046,8 +1050,11 @@ export function ScrapeFilesDownloadForm({
download,
submitter,
clearOnSuccess,
+ onDelete,
onCancel,
+ onSuccess,
actions = DownloadFormButtons,
+ singleDownload = false,
}) {
const [showMessage, setShowMessage] = React.useState(false);
@@ -1055,7 +1062,7 @@ export function ScrapeFilesDownloadForm({
const downloadData = {
downloader: Downloaders.ScrapeHtml,
sub_downloader: Downloaders.File,
- tag_names: formData.tag_names,
+ tag_names: formData.tag_names || [],
destination: formData.destination,
urls: formData.urls.split(/\r?\n/),
settings: formData.settings,
@@ -1074,15 +1081,28 @@ export function ScrapeFilesDownloadForm({
}
};
+ const localOnSuccess = async () => {
+ setShowMessage(true);
+ if (onSuccess) {
+ onSuccess();
+ }
+ }
+
const form = useForm({
submitter,
defaultFormData: mergeDeep(emptyFormData, download),
emptyFormData,
clearOnSuccess,
- onSuccess: async () => setShowMessage(true),
+ onSuccess: localOnSuccess,
});
- const actionsElm = actions({onCancel, form});
+ const urlInput = singleDownload ?
+
+ : ;
+
+ // Default to "new" download buttons.
+ actions = actions || DownloadFormButtons;
+ const actionsElm = actions({onDelete, onCancel, form});
return
- {showMessage &&
-
-
-
-
- }
+ {showMessage && }
{actionsElm}
@@ -1125,6 +1140,30 @@ export function ScrapeFilesDownloadForm({
}
+export function EditScrapeFilesDownloadForm({download, onDelete, onCancel, onSuccess}) {
+ const submitter = async (formData) => {
+ const downloadData = {
+ downloader: Downloaders.ScrapeHtml,
+ sub_downloader: Downloaders.File,
+ tag_names: formData.tag_names || [],
+ destination: formData.destination,
+ urls: [formData.url],
+ settings: formData.settings,
+ }
+ await postDownload(downloadData);
+ };
+
+ return
+}
+
export function DownloadMenu({onOpen, disabled}) {
const [downloader, setDownloader] = useState();
diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js
index 0ec9b4c0..9723eab3 100644
--- a/app/src/components/admin/Downloads.js
+++ b/app/src/components/admin/Downloads.js
@@ -31,7 +31,12 @@ import {
} from "semantic-ui-react";
import {Button, Header, Modal, ModalActions, ModalContent, ModalHeader, Placeholder, Segment, Table} from "../Theme";
import {useDownloads} from "../../hooks/customHooks";
-import {EditChannelDownloadForm, EditRSSDownloadForm, EditZimDownloadForm} from "../Download";
+import {
+ EditChannelDownloadForm,
+ EditRSSDownloadForm,
+ EditScrapeFilesDownloadForm,
+ EditZimDownloadForm
+} from "../Download";
import {Downloaders} from "../Vars";
function ClearDownloadsButton({callback}) {
@@ -200,6 +205,13 @@ function RecurringDownloadRow({download, fetchDownloads, onDelete}) {
onCancel={handleEditClose}
onSuccess={onSuccess}
/>;
+ } else if (downloader === Downloaders.ScrapeHtml) {
+ editForm = ;
}
const editModal = {
const fetchArchive = async () => {
try {
- const [file_group, history] = await getArchive(archiveId);
+ const {file_group, history} = await getArchive(archiveId);
setArchiveFileGroup(file_group);
setHistory(history);
} catch (e) {
diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py
index 58056446..9d75035f 100644
--- a/modules/videos/channel/api.py
+++ b/modules/videos/channel/api.py
@@ -107,34 +107,6 @@ def channel_refresh(_: Request, channel_id: int):
return response.empty()
-@channel_bp.post('//download', name='post_channel_download')
-@channel_bp.put('//download/', name='put_channel_download')
-@openapi.description('Create or update a Channel Download record')
-@validate(schema.ChannelDownloadRequest)
-@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
-@wrol_mode_check
-async def channel_download(_: Request, channel_id: int, body: schema.ChannelDownloadRequest, download_id: int = None):
- if download_id:
- await lib.update_channel_download(
- channel_id,
- download_id,
- body.url,
- body.frequency,
- body.settings,
- )
- else:
- await lib.create_channel_download(
- channel_id,
- body.url,
- body.frequency,
- body.settings,
- )
- if download_manager.is_disabled:
- # Warn the user that downloads are disabled.
- Events.send_downloads_disabled('Channel download created. But downloads are disabled.')
- return response.empty()
-
-
@channel_bp.post('//tag')
@openapi.definition(
description='Tag/untag a Channel with a single tag',
diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py
index 46dece53..84a89524 100644
--- a/modules/videos/channel/test/test_api.py
+++ b/modules/videos/channel/test/test_api.py
@@ -1,4 +1,3 @@
-import asyncio
import json
import pathlib
import shutil
@@ -10,6 +9,7 @@ import mock
import pytest
from modules.videos.common import get_videos_directory
+from modules.videos.conftest import simple_channel
from modules.videos.downloader import ChannelDownloader
from modules.videos.lib import save_channels_config, get_channels_config
from modules.videos.models import Channel, Video
@@ -71,7 +71,7 @@ def test_channel_no_download_frequency(test_client, test_session, test_directory
# Update the Channel with a frequency.
new_download = {'urls': [simple_channel.url], 'frequency': 10, 'downloader': ChannelDownloader.name}
request, response = test_client.post('/api/download', json=new_download)
- assert response.status_code == HTTPStatus.NO_CONTENT, response.json
+ assert response.status_code == HTTPStatus.CREATED, response.json
# Download is scheduled. Download is related to Channel.
assert len(download_manager.get_downloads(test_session)) == 1
@@ -410,46 +410,47 @@ async def test_get_channel_videos_pagination(test_session, simple_channel, video
@pytest.mark.asyncio
-async def test_create_channel_download_api(async_client, test_session, simple_channel, tag_factory):
- tag = await tag_factory()
+async def test_channel_download_id(async_client, test_session, tag_factory, simple_channel, test_download_manager,
+ test_downloader):
+ tag = await tag_factory()
- # Create Download.
- body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
- request, response = await async_client.post(
- f'/api/videos/channels/{simple_channel.id}/download', json=body)
- assert response.status_code == HTTPStatus.NO_CONTENT, response.content
- download: Download = test_session.query(Download).one()
- assert download.url == 'https://example.com/1' and not download.settings.get('tag_names')
- assert download.settings.get('title_include') is None
+ # A recurring Download can be displayed on a Channel's page.
+ body = dict(
+ urls=['https://example.com/channel1'],
+ tag_names=[tag.name],
+ downloader=test_downloader.name,
+ settings=dict(channel_id=simple_channel.id),
+ frequency=120,
+ )
+ request, response = await async_client.post(f'/api/download', json=body)
+ assert response.status_code == HTTPStatus.CREATED
+ download = test_session.query(Download).one()
+ assert download.url == 'https://example.com/channel1'
+ assert download.channel_id == simple_channel.id
- # Create another Download. Previous Downloads is untouched.
- body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}}
- request, response = await async_client.post(
- f'/api/videos/channels/{simple_channel.id}/download', json=body)
- assert response.status_code == HTTPStatus.NO_CONTENT, response.content
- download1, download2 = test_session.query(Download).order_by(Download.url).all()
- assert download1.url == 'https://example.com/1' and not download1.settings.get('tag_names')
- assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ]
- test_session.flush(simple_channel)
- assert len(simple_channel.downloads) == 2
-
- # Change frequency of first Download.
- body = {'url': 'https://example.com/1', 'frequency': 123, 'settings': {'title_include': 'foo,bar'}}
- request, response = await async_client.put(
- f'/api/videos/channels/{simple_channel.id}/download/{download1.id}', json=body)
- assert response.status_code == HTTPStatus.NO_CONTENT, response.content
- download1, download2 = test_session.query(Download).order_by(Download.url).all()
- assert download1.url == 'https://example.com/1'
- assert download1.settings == {'title_include': 'foo,bar'}
- assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ]
- test_session.flush(simple_channel)
- assert len(simple_channel.downloads) == 2
-
- # Deleting Download does not delete Channel.
- request, response = await async_client.delete(f'/api/download/{download1.id}')
+ # A Channel relationship can be removed from a Download.
+ body = dict(
+ urls=['https://example.com/channel1'],
+ tag_names=[tag.name],
+ downloader=test_downloader.name,
+ settings=dict(),
+ frequency=240,
+ )
+ request, response = await async_client.put(f'/api/download/{download.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
- assert test_session.query(Download).count() == 1
- assert test_session.query(Channel).count() == 1
+ download = test_session.query(Download).one()
+ assert download.channel_id is None
+ assert download.frequency == 240
+
+ # A once-Download cannot be associated with a Channel.
+ body = dict(
+ urls=['https://example.com/channel1'],
+ tag_names=[tag.name],
+ downloader=test_downloader.name,
+ settings=dict(channel_id=simple_channel.id),
+ )
+ request, response = await async_client.put(f'/api/download/{download.id}', json=body)
+ assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py
index d51f5582..843c9e16 100644
--- a/modules/videos/test/test_api.py
+++ b/modules/videos/test/test_api.py
@@ -151,7 +151,7 @@ def test_api_download(test_session, test_client, test_directory):
content = dict(urls=['https://example.com/video1', ], downloader='video', destination='dest',
settings=dict(excluded_urls='example.com'))
request, response = test_client.post('/api/download', content=json.dumps(content))
- assert response.status_code == HTTPStatus.NO_CONTENT
+ assert response.status_code == HTTPStatus.CREATED
download = test_session.query(Download).one()
assert download.url == 'https://example.com/video1'
@@ -167,15 +167,14 @@ async def test_api_download_video_tags(test_session, async_client, tag_factory):
tag1 = await tag_factory()
tag2 = await tag_factory()
- content = dict(urls=['https://example.com/video1', ], downloader='video',
- settings={'tag_names': [tag1.name, tag2.name]})
+ content = dict(urls=['https://example.com/video1', ], downloader='video', tag_names=[tag1.name, tag2.name])
request, response = await async_client.post('/api/download', content=json.dumps(content))
- assert response.status_code == HTTPStatus.NO_CONTENT
+ assert response.status_code == HTTPStatus.CREATED
download = test_session.query(Download).one()
assert download.url == 'https://example.com/video1'
assert download.downloader == 'video'
- assert download.settings['tag_names'] == [tag1.name, tag2.name]
+ assert download.tag_names == [tag1.name, tag2.name]
@pytest.mark.asyncio
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index 744bbcca..547ada21 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -453,7 +453,7 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac
@pytest.mark.asyncio
async def test_channel_download_crud(test_session, async_client, assert_downloads, tag_factory):
- """Test creatin"""
+ """Test creating more complex Channel Download."""
one, two = await tag_factory(), await tag_factory()
download = {
'urls': ['https://www.youtube.com/@wrolpi/videos'],
@@ -476,10 +476,11 @@ async def test_channel_download_crud(test_session, async_client, assert_download
}
request, response = await async_client.post('/api/download', json=download)
- assert response.status == HTTPStatus.NO_CONTENT
+ assert response.status == HTTPStatus.CREATED, request.body
download = test_session.query(Download).one()
assert download.url == 'https://www.youtube.com/@wrolpi/videos'
+ assert download.destination is None
assert download.tag_names == ['one', 'two']
assert download.downloader == 'video_channel'
assert download.sub_downloader == 'video'
diff --git a/modules/zim/schema.py b/modules/zim/schema.py
index 046a382b..d6042d2a 100644
--- a/modules/zim/schema.py
+++ b/modules/zim/schema.py
@@ -5,7 +5,7 @@ from typing import Optional, List, Dict
@dataclass
class ZimSearchRequest:
search_str: Optional[str] = None
- tag_names: List[str] = field(default_factory=lambda: list())
+ tag_names: List[str] = field(default_factory=list)
offset: Optional[int] = 0
limit: Optional[int] = 10
@@ -98,5 +98,5 @@ class OutdatedZims:
@dataclass
class SearchEstimateRequest:
search_str: Optional[str] = None
- tag_names: List[str] = field(default_factory=lambda: list())
+ tag_names: List[str] = field(default_factory=list)
any_tag: bool = False
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index 77388864..b8e800ef 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -584,6 +584,10 @@ class DownloadManager:
if not frequency or not isinstance(frequency, int):
raise ValueError('Recurring download must have a frequency!')
+ from wrolpi.scrape_downloader import ScrapeHTMLDownloader
+ if downloader_name == ScrapeHTMLDownloader.name and frequency:
+ raise InvalidDownload(f'Cannot schedule recurring download for {downloader_name=}')
+
download, = self.create_downloads([url, ], session=session, downloader_name=downloader_name,
reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
destination=destination, tag_names=tag_names, settings=settings)
@@ -601,17 +605,22 @@ class DownloadManager:
@optional_session
def update_download(self, id_: int, url: str, downloader: str,
destination: str | pathlib.Path = None, tag_names: List[str] = None,
- sub_downloader: str | None = None,
+ sub_downloader: str | None = None, frequency: int = None,
settings: Dict = None, session: Session = None) -> Download:
download = self.find_download(session=session, id_=id_)
+ if settings and settings.get('channel_id') and not frequency:
+ raise InvalidDownload(f'A once-download cannot be associated with a Channel')
download.url = url
download.downloader = downloader
+ download.frequency = frequency
# Preserve existing settings, unless new settings are provided.
download.settings = settings if settings is not None else download.settings
# Use provided params even if empty.
download.destination = destination or None
download.tag_names = tag_names or None
download.sub_downloader = sub_downloader or None
+ # Remove Channel relationship, if necessary.
+ download.channel_id = (settings or dict()).get('channel_id')
return download
@wrol_mode_check
diff --git a/wrolpi/files/test/test_scrape_downloader.py b/wrolpi/files/test/test_scrape_downloader.py
index fddefe4b..3976d817 100644
--- a/wrolpi/files/test/test_scrape_downloader.py
+++ b/wrolpi/files/test/test_scrape_downloader.py
@@ -1,7 +1,10 @@
import asyncio
+from http import HTTPStatus
import mock
+import pytest
+from wrolpi.dates import Seconds
from wrolpi.downloader import DownloadResult
from wrolpi.files.downloader import FileDownloader
from wrolpi.scrape_downloader import ScrapeHTMLDownloader
@@ -76,3 +79,18 @@ async def test_scrape_html_downloader_html(test_directory, test_session, test_do
'https://example.com/dir',
'https://example.com/dir/other.html',
])
+
+
+@pytest.mark.asyncio
+async def test_scrape_html_downloader_api(async_client, test_session, test_download_manager):
+ """Scrape Download cannot be recurring."""
+ body = dict(
+ urls=['https://example.com'],
+ destination='uploads',
+ downloader='scrape_html',
+ sub_downloader='file',
+ frequency=Seconds.week,
+ tag_names=[],
+ settings=dict(depth=1, max_pages=1, suffix='.mp4'))
+ request, response = await async_client.post('/api/download', json=body)
+ assert response.status == HTTPStatus.BAD_REQUEST
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index f7d6604a..4fa6a349 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -255,13 +255,15 @@ def valid_regex(_: Request, body: schema.RegexRequest):
@api_bp.post('/download')
-@openapi.description('Download all the URLs that are provided.')
-@validate(schema.DownloadRequest)
+@openapi.definition(
+ description='Download all the URLs that are provided.',
+ body=schema.DownloadRequest,
+ validate=True,
+)
@wrol_mode_check
async def post_download(_: Request, body: schema.DownloadRequest):
- downloader = download_manager.find_downloader_by_name(body.downloader)
- if not downloader:
- raise InvalidDownload(f'Cannot find downloader with name {body.downloader}')
+ # Raises an InvalidDownload if the Downloader cannot be found.
+ download_manager.find_downloader_by_name(body.downloader)
kwargs = dict(downloader_name=body.downloader,
sub_downloader_name=body.sub_downloader, reset_attempts=True,
@@ -275,17 +277,20 @@ async def post_download(_: Request, body: schema.DownloadRequest):
# Downloads are disabled, warn the user.
Events.send_downloads_disabled('Download created. But, downloads are disabled.')
- return response.empty()
+ return response.empty(status=HTTPStatus.CREATED)
@api_bp.put('/download/')
-@openapi.description('Update properties of the Download')
-@validate(schema.DownloadRequest)
+@openapi.definition(
+ description='Update properties of the Download',
+ body=schema.DownloadRequest,
+ validate=True,
+)
@wrol_mode_check
async def put_download(_: Request, download_id: int, body: schema.DownloadRequest):
- downloader = download_manager.find_downloader_by_name(body.downloader)
- if not downloader:
- raise ValidationError(f'Cannot find downloader with name {body.downloader}')
+ # Raises an InvalidDownload if the Downloader cannot be found.
+ download_manager.find_downloader_by_name(body.downloader)
+
if len(body.urls) != 1:
raise ValidationError('Only one URL can be specified when updating a Download')
@@ -295,6 +300,7 @@ async def put_download(_: Request, download_id: int, body: schema.DownloadReques
url=body.urls[0],
downloader=body.downloader,
destination=body.destination,
+ frequency=body.frequency,
tag_names=body.tag_names,
sub_downloader=body.sub_downloader,
settings=body.settings,
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index f206cd3b..0a5a75ec 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -80,8 +80,8 @@ class EchoResponse:
@dataclass
class DownloadSettings:
- channel_id: int = None
- channel_tag_name: List[str] = field(default_factory=lambda: list())
+ channel_id: Optional[int] = None
+ channel_tag_name: List[str] = field(default_factory=list)
depth: Optional[int] = None
download_metadata_only: bool = False
download_order: Optional[str] = None
@@ -90,16 +90,13 @@ class DownloadSettings:
maximum_duration: Optional[int] = None
minimum_duration: Optional[int] = None
suffix: Optional[str] = None
- tag_names: List[str] = field(default_factory=lambda: list())
- title_exclude: str = None
- title_include: str = None
+ title_exclude: Optional[str] = None
+ title_include: Optional[str] = None
video_count_limit: Optional[int] = None
- video_format: str = None
- video_resolutions: List[str] = field(default_factory=lambda: list())
+ video_format: Optional[str] = None
+ video_resolutions: List[str] = field(default_factory=list)
def __post_init__(self):
- self.tag_names = [i.strip() for i in self.tag_names] if self.tag_names else []
-
if self.excluded_urls and self.excluded_urls.endswith(','):
raise ValidationError('Excluded urls cannot end with ,')
if self.excluded_urls and self.excluded_urls.startswith(','):
@@ -128,7 +125,7 @@ class DownloadRequest:
urls: List[str]
downloader: str
destination: Optional[str] = None
- tag_names: List[str] = field(default_factory=list)
+ tag_names: Optional[List[str]] = field(default_factory=list)
frequency: Optional[int] = None
sub_downloader: Optional[str] = None
settings: Optional[dict] = field(default_factory=dict)
@@ -146,6 +143,8 @@ class DownloadRequest:
destination = get_relative_to_media_directory(destination)
self.destination = str(destination)
+ self.tag_names = self.tag_names or []
+
# Validate settings contents. Remove empty values.
settings = self.settings or dict()
self.settings = {k: v for k, v in DownloadSettings(**settings).__dict__.items() if v not in ([], None)}
@@ -226,8 +225,8 @@ class SearchSuggestionsRequest:
@dataclass
class SearchFileEstimateRequest:
search_str: Optional[str] = None
- mimetypes: List[str] = field(default_factory=lambda: list())
- tag_names: List[str] = field(default_factory=lambda: list())
+ mimetypes: List[str] = field(default_factory=list)
+ tag_names: List[str] = field(default_factory=list)
months: Optional[List[int]] = None
from_year: Optional[int] = None
to_year: Optional[int] = None
@@ -240,7 +239,7 @@ class SearchFileEstimateRequest:
@dataclass
class SearchOtherEstimateRequest:
- tag_names: List[str] = field(default_factory=lambda: list())
+ tag_names: List[str] = field(default_factory=list)
@dataclass
diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py
index d391bea3..d60dfa27 100644
--- a/wrolpi/test/test_downloader.py
+++ b/wrolpi/test/test_downloader.py
@@ -382,7 +382,7 @@ async def test_crud_download(async_client, test_session, test_download_manager,
settings=dict(excluded_urls='example.org,something'),
)
request, response = await async_client.post('/api/download', content=json.dumps(body))
- assert response.status_code == HTTPStatus.NO_CONTENT, response.body
+ assert response.status_code == HTTPStatus.CREATED, response.body
download: Download = test_session.query(Download).one()
assert download.id
diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py
index dd2b571c..fb1ef349 100644
--- a/wrolpi/test/test_root_api.py
+++ b/wrolpi/test/test_root_api.py
@@ -430,42 +430,46 @@ async def test_download_crud(test_session, async_client, test_download_manager_c
with mock.patch('wrolpi.downloader.DownloadManager.dispatch_downloads', dispatch_downloads):
# Create a single recurring Download.
- content = dict(
+ body = dict(
urls=['https://example.com/1', ],
frequency=1_000,
downloader='archive',
+ tag_names=None, # tag_names can be a None.
)
- request, response = await async_client.post('/api/download', content=json.dumps(content))
- assert response.status_code == HTTPStatus.NO_CONTENT
+ request, response = await async_client.post('/api/download', content=json.dumps(body))
+ assert response.status_code == HTTPStatus.CREATED
assert {i.url for i in test_session.query(Download)} == {'https://example.com/1', }
# Create two once-downloads.
- content = dict(
+ body = dict(
urls=['https://example.com/2', 'https://example.com/3'],
downloader='archive',
+ # tag_names=None, # tag_names can be missing.
)
- request, response = await async_client.post('/api/download', json=content)
- assert response.status_code == HTTPStatus.NO_CONTENT
+ request, response = await async_client.post('/api/download', json=body)
+ assert response.status_code == HTTPStatus.CREATED
assert {i.url for i in test_session.query(Download)} == {f'https://example.com/{i}' for i in range(1, 4)}
download1, download2, download3 = test_session.query(Download).order_by(Download.url).all()
assert not download1.settings
- expected_download1 = dict(download1.__json__()).copy()
+ expected_download = dict(download1.__json__()).copy()
# Update a download.
- content = dict(
+ body = dict(
urls=['https://example.com/1'],
downloader='archive',
- settings={'tag_names': [tag1.name, tag2.name]},
+ frequency=1_000,
+ tag_names=[tag1.name, tag2.name],
)
- expected_download1.update({'url': 'https://example.com/1', 'settings': {'tag_names': [tag1.name, tag2.name]}})
- request, response = await async_client.put(f'/api/download/{download1.id}', json=content)
+ expected_download.update({'url': 'https://example.com/1', 'tag_names': [tag1.name, tag2.name]})
+ request, response = await async_client.put(f'/api/download/{download1.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
test_session.flush([download2])
- download2 = test_session.query(Download).filter_by(id=expected_download1['id']).first()
- assert download2.__json__() == expected_download1
+ download2 = test_session.query(Download).filter_by(id=expected_download['id']).first()
+ assert download2.__json__() == expected_download
+ assert download2.tag_names == [tag1.name, tag2.name]
def test_get_downloaders(test_client):
@@ -514,33 +518,6 @@ async def test_restart_download(test_session, async_client, test_download_manage
assert download.is_deferred, download.status
-@pytest.mark.asyncio
-async def test_put_download(test_session, async_client, test_download_manager, test_downloader, channel_factory):
- """A Download can be updated using a PUT request."""
- channel = channel_factory()
- download = test_download_manager.create_download('https://example.com', test_downloader.name)
- assert not download.settings
-
- body = dict(
- downloader=test_downloader.name,
- urls=['https://example.com/new-url'], # URL can be changed.
- settings=dict(channel_id=channel.id),
- )
- request, response = await async_client.put(f'/api/download/{download.id}', json=body)
- assert response.status_code == HTTPStatus.NO_CONTENT
- assert test_session.query(Download).count() == 1, 'Download should have been updated'
- download = test_session.query(Download).one()
- assert download.url == 'https://example.com/new-url', 'Download URL was not changed'
- assert download.settings == dict(channel_id=channel.id), 'Download settings were not changed'
-
- # Settings can be cleared by sending `None`
- body['settings'] = None
- request, response = await async_client.put(f'/api/download/{download.id}', json=body)
- assert response.status_code == HTTPStatus.NO_CONTENT
- download = test_session.query(Download).one()
- assert download.settings == dict(), 'Download settings were not changed'
-
-
def test_get_global_statistics(test_session, test_client):
request, response = test_client.get('/api/statistics')
assert response.json['global_statistics']['db_size'] > 1