diff --git a/app/src/api.js b/app/src/api.js index 743bc8a1..6f7c5fcc 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -413,74 +413,6 @@ export async function getVideosStatistics() { return response } -export async function createChannelDownload(channelId, url, frequency, title_include, title_exclude, tag_names) { - const body = { - url: url, - frequency: frequency, - settings: { - title_include: title_include, - title_exclude: title_exclude, - tag_names: tag_names, - }, - } - url = `${VIDEOS_API}/channels/${channelId}/download`; - const response = await apiPost(url, body); - if (!response.ok) { - const json = await response.json(); - if (json['code'] === 'INVALID_DOWNLOAD') { - toast({ - type: 'error', - title: 'Cannot Download!', - description: 'This channel does not have a download record. Modify the frequency.', - time: 5000, - }); - } else { - const message = await getErrorMessage(response, 'Could not create Channel download.'); - toast({ - type: 'error', - title: 'Creating Channel Download Failed', - description: message, - time: 5000, - }); - } - } - return response; -} - -export async function updateChannelDownload(channelId, downloadId, url, frequency, title_include, title_exclude, tag_names) { - const body = { - url: url, - frequency: frequency, - settings: { - title_include: title_include, - title_exclude: title_exclude, - tag_names: tag_names, - }, - } - url = `${VIDEOS_API}/channels/${channelId}/download/${downloadId}`; - const response = await apiPut(url, body); - if (!response.ok) { - const json = await response.json(); - if (json['code'] === 'INVALID_DOWNLOAD') { - toast({ - type: 'error', - title: 'Cannot Download!', - description: 'This channel does not have a download record. Modify the frequency.', - time: 5000, - }); - } else { - const message = await getErrorMessage(response, 'Could not update Channel download.'); - toast({ - type: 'error', - title: 'Updating Channel Download Failed', - description: message, - time: 5000, - }); - } - } - return response; -} - export async function refreshChannel(channelId) { let url = `${VIDEOS_API}/channels/refresh/${channelId}`; const response = await apiPost(url); @@ -744,7 +676,10 @@ export async function getArchive(archiveId) { const response = await apiGet(`${ARCHIVES_API}/${archiveId}`); if (response.ok) { const data = await response.json(); - return [data['file_group'], data['history']]; + return { + file_group: data['file_group'], + history: data['history'], + } } else { const message = await getErrorMessage(response, 'Unable to get Archive. See server logs.'); toast({ @@ -764,7 +699,7 @@ export async function postDownload(downloadData) { throw new Error('downloader is required, but was not provided'); } - let body = { + const body = { urls: downloadData.urls, destination: downloadData.destination, downloader: downloadData.downloader, diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 60780617..8417a730 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -1865,7 +1865,7 @@ export function mergeDeep(target, source) { if (Array.isArray(source[key])) { // source overwrites target if it is a list with values. result[key] = source[key] || target[key]; - } else if (typeof source[key] === 'object' && typeof target[key] === 'object') { + } else if (typeof source[key] === 'object' && typeof target[key] === 'object' && source[key] !== null) { result[key] = mergeDeep(target[key] || {}, source[key]); } else { result[key] = source[key] !== undefined ? source[key] : target[key]; diff --git a/app/src/components/Download.cy.js b/app/src/components/Download.cy.js index 08868d33..39195b35 100644 --- a/app/src/components/Download.cy.js +++ b/app/src/components/Download.cy.js @@ -1,5 +1,5 @@ import React from 'react'; -import {ArchiveDownloadForm, ChannelDownloadForm} from './Download'; +import {ArchiveDownloadForm, ChannelDownloadForm, EditRSSDownloadForm} from './Download'; import 'semantic-ui-offline/semantic.min.css'; describe('', () => { @@ -66,3 +66,87 @@ describe('', () => { .get('.item').contains('180 Days').click(); }); }); + +const downloadersResponse = { + "downloaders": [{"name": "archive", "pretty_name": "Archive"}, { + "name": "video", + "pretty_name": "Videos" + }], "manager_disabled": true +}; + +describe('', () => { + beforeEach(() => { + const download = { + id: 1, + destination: "destination directory", + downloader: "rss", + frequency: 604800, + next_download: "2024-11-24T16:34:21.014400+00:00", + settings: { + title_include: "included", + title_exclude: "excluded", + video_format: "mp4", + video_resolutions: [ + "720p", + "480p", + "maximum" + ] + }, + sub_downloader: "video", + tag_names: ['Automotive'], + url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC4t8bw1besFTyjW7ZBCOIrw" + }; + cy.mountWithTags(); + cy.intercept('GET', '/api/downloaders', downloadersResponse).as('downloaders'); + cy.intercept('POST', '**/api/files/search_directories', { + statusCode: 200, + body: { + is_dir: true, channel_directories: [], domain_directories: [], + directories: [{path: "some directory", name: "some directory"}], + }, + }).as('searchDirectories'); + + }); + + it('can edit videos download', () => { + // Weekly frequency. + cy.get('#url_input').should('have.value', + 'https://www.youtube.com/feeds/videos.xml?channel_id=UC4t8bw1besFTyjW7ZBCOIrw'); + cy.get('div.divider.text').contains('Weekly'); + cy.get('input[name="title_include"]').should('have.value', 'included'); + cy.get('input[name="title_exclude"]').should('have.value', 'excluded'); + cy.get('#destination_search_form').should('be.visible'); + cy.get('div.ui.large.label').contains('Automotive').should('be.visible'); + }); + + it('can submit videos download', () => { + cy.get('button[type="submit"]').should('be.visible'); + + cy.intercept('PUT', '/api/download/1', (req) => { + expect(req.body).to.deep.equal(JSON.stringify({ + "destination": "destination directory", + "downloader": "rss", + "frequency": 604800, + "settings": { + "excluded_urls": null, + "title_exclude": "excluded", + "title_include": "included", + "video_resolutions": [ + "720p", + "480p", + "maximum" + ], + "video_format": "mp4", + "minimum_duration": null, + "maximum_duration": null + }, + "sub_downloader": "video", + "tag_names": ["Automotive"], + "urls": ["https://www.youtube.com/feeds/videos.xml?channel_id=UC4t8bw1besFTyjW7ZBCOIrw"] + })); + req.reply({statusCode: 204}); + }).as('downloadPut'); + + cy.get('button[type="submit"]').click(); + }); +}); diff --git a/app/src/components/Download.js b/app/src/components/Download.js index 9eb11ae8..e86d77e6 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -59,10 +59,11 @@ export function DestinationForm({ form, helpContent, label = 'Destination', + name = 'destination', path = 'destination', required = false, }) { - const [inputProps, inputAttrs] = form.getCustomProps({name: 'destination', path, required}); + const [inputProps, inputAttrs] = form.getCustomProps({name, path, required}); const {disabled, value, onChange} = inputProps; const helpPopup = helpContent ? : null; return @@ -72,6 +73,7 @@ export function DestinationForm({ value={value} onSelect={onChange} disabled={disabled} + id='destination_search_form' /> } @@ -129,6 +131,7 @@ export function DownloaderSelector({form, name = 'sub_downloader', path = 'sub_d label='Downloader' placeholder='Select a downloader' options={downloaders} + id='downloader_selector' {...inputProps} /> } @@ -339,6 +342,8 @@ export function ChannelTagNameForm({form}) { } export function VideosDownloadForm({singleDownload = true, onCancel}) { + const [showMessage, setShowMessage] = React.useState(false); + // Keep video settings in session to help user start downloads consistently. const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions); const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption); @@ -366,10 +371,15 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) { } const onSuccess = () => { + setShowMessage(true); form.reset(); } - const form = useForm({submitter, defaultFormData, onSuccess}); + const form = useForm({ + submitter, + defaultFormData, + onSuccess, + }); React.useEffect(() => { const {video_resolutions, video_format} = form.formData.settings; @@ -432,6 +442,7 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) { + {showMessage && } @@ -476,6 +487,7 @@ export function ChannelDownloadForm({ clearOnSuccess = true, channel_id = null, }) { + const [showMessage, setShowMessage] = React.useState(false); // May have received submitter from EditChannelDownloadForm. submitter = submitter || (async (formData) => { @@ -491,6 +503,13 @@ export function ChannelDownloadForm({ await postDownload(downloadData); }); + const localOnSuccess = async () => { + setShowMessage(true); + if (onSuccess) { + onSuccess(); + } + } + // Keep video settings in session to help user start downloads consistently. const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions); const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption); @@ -520,7 +539,7 @@ export function ChannelDownloadForm({ submitter, defaultFormData: mergeDeep(emptyFormData, download), emptyFormData, - onSuccess, + onSuccess: localOnSuccess, clearOnSuccess, }); @@ -608,6 +627,7 @@ export function ChannelDownloadForm({ + {showMessage && } {actionsElm} @@ -632,7 +652,7 @@ export function EditChannelDownloadForm({ downloader: formData.downloader, frequency: formData.frequency, settings: formData.settings, - tag_names: formData.tag_names, + tag_names: formData.tag_names || [], urls: [formData.url,], } await putDownload(download.id, downloadData); @@ -651,12 +671,15 @@ export function EditChannelDownloadForm({ } function SuccessfulDownloadSubmitMessage() { - return - Download Submitted - - View downloads - - + return + + Download Submitted + + View downloads + + + + } export function ArchiveDownloadForm({download, onCancel}) { @@ -700,12 +723,7 @@ export function ArchiveDownloadForm({download, onCancel}) { - {showMessage && - - - - - } + {showMessage && } @@ -728,7 +746,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action frequency: formData.frequency, settings: formData.settings, sub_downloader: formData.sub_downloader, - tag_names: formData.tag_names, + tag_names: formData.tag_names || [], urls: [formData.url], } await postDownload(downloadData); @@ -840,12 +858,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action {downloaderRows} - {showMessage && - - - - - } + {showMessage && } {actionsElm} @@ -859,11 +872,12 @@ export function EditRSSDownloadForm({download, onDelete, onCancel, actions = Edi const submitter = async (formData) => { const downloadData = { + destination: formData.destination, downloader: formData.downloader, frequency: formData.frequency, settings: formData.settings, sub_downloader: formData.sub_downloader, - tag_names: formData.tag_names, + tag_names: formData.tag_names || [], urls: [formData.url,], } await putDownload(download.id, downloadData); @@ -927,12 +941,7 @@ export function EditZimDownloadForm({download, onDelete, onCancel, actions = Edi - {showMessage && - - - - - } + {showMessage && } {actionsElm} @@ -997,12 +1006,7 @@ export function FilesDownloadForm({ - {showMessage && - - - - - } + {showMessage && } {actionsElm} @@ -1046,8 +1050,11 @@ export function ScrapeFilesDownloadForm({ download, submitter, clearOnSuccess, + onDelete, onCancel, + onSuccess, actions = DownloadFormButtons, + singleDownload = false, }) { const [showMessage, setShowMessage] = React.useState(false); @@ -1055,7 +1062,7 @@ export function ScrapeFilesDownloadForm({ const downloadData = { downloader: Downloaders.ScrapeHtml, sub_downloader: Downloaders.File, - tag_names: formData.tag_names, + tag_names: formData.tag_names || [], destination: formData.destination, urls: formData.urls.split(/\r?\n/), settings: formData.settings, @@ -1074,15 +1081,28 @@ export function ScrapeFilesDownloadForm({ } }; + const localOnSuccess = async () => { + setShowMessage(true); + if (onSuccess) { + onSuccess(); + } + } + const form = useForm({ submitter, defaultFormData: mergeDeep(emptyFormData, download), emptyFormData, clearOnSuccess, - onSuccess: async () => setShowMessage(true), + onSuccess: localOnSuccess, }); - const actionsElm = actions({onCancel, form}); + const urlInput = singleDownload ? + + : ; + + // Default to "new" download buttons. + actions = actions || DownloadFormButtons; + const actionsElm = actions({onDelete, onCancel, form}); return
Scrape Files
@@ -1091,7 +1111,7 @@ export function ScrapeFilesDownloadForm({ - + {urlInput} @@ -1110,12 +1130,7 @@ export function ScrapeFilesDownloadForm({ - {showMessage && - - - - - } + {showMessage && } {actionsElm} @@ -1125,6 +1140,30 @@ export function ScrapeFilesDownloadForm({ } +export function EditScrapeFilesDownloadForm({download, onDelete, onCancel, onSuccess}) { + const submitter = async (formData) => { + const downloadData = { + downloader: Downloaders.ScrapeHtml, + sub_downloader: Downloaders.File, + tag_names: formData.tag_names || [], + destination: formData.destination, + urls: [formData.url], + settings: formData.settings, + } + await postDownload(downloadData); + }; + + return +} + export function DownloadMenu({onOpen, disabled}) { const [downloader, setDownloader] = useState(); diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js index 0ec9b4c0..9723eab3 100644 --- a/app/src/components/admin/Downloads.js +++ b/app/src/components/admin/Downloads.js @@ -31,7 +31,12 @@ import { } from "semantic-ui-react"; import {Button, Header, Modal, ModalActions, ModalContent, ModalHeader, Placeholder, Segment, Table} from "../Theme"; import {useDownloads} from "../../hooks/customHooks"; -import {EditChannelDownloadForm, EditRSSDownloadForm, EditZimDownloadForm} from "../Download"; +import { + EditChannelDownloadForm, + EditRSSDownloadForm, + EditScrapeFilesDownloadForm, + EditZimDownloadForm +} from "../Download"; import {Downloaders} from "../Vars"; function ClearDownloadsButton({callback}) { @@ -200,6 +205,13 @@ function RecurringDownloadRow({download, fetchDownloads, onDelete}) { onCancel={handleEditClose} onSuccess={onSuccess} />; + } else if (downloader === Downloaders.ScrapeHtml) { + editForm = ; } const editModal = { const fetchArchive = async () => { try { - const [file_group, history] = await getArchive(archiveId); + const {file_group, history} = await getArchive(archiveId); setArchiveFileGroup(file_group); setHistory(history); } catch (e) { diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index 58056446..9d75035f 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -107,34 +107,6 @@ def channel_refresh(_: Request, channel_id: int): return response.empty() -@channel_bp.post('//download', name='post_channel_download') -@channel_bp.put('//download/', name='put_channel_download') -@openapi.description('Create or update a Channel Download record') -@validate(schema.ChannelDownloadRequest) -@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) -@wrol_mode_check -async def channel_download(_: Request, channel_id: int, body: schema.ChannelDownloadRequest, download_id: int = None): - if download_id: - await lib.update_channel_download( - channel_id, - download_id, - body.url, - body.frequency, - body.settings, - ) - else: - await lib.create_channel_download( - channel_id, - body.url, - body.frequency, - body.settings, - ) - if download_manager.is_disabled: - # Warn the user that downloads are disabled. - Events.send_downloads_disabled('Channel download created. But downloads are disabled.') - return response.empty() - - @channel_bp.post('//tag') @openapi.definition( description='Tag/untag a Channel with a single tag', diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index 46dece53..84a89524 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -1,4 +1,3 @@ -import asyncio import json import pathlib import shutil @@ -10,6 +9,7 @@ import mock import pytest from modules.videos.common import get_videos_directory +from modules.videos.conftest import simple_channel from modules.videos.downloader import ChannelDownloader from modules.videos.lib import save_channels_config, get_channels_config from modules.videos.models import Channel, Video @@ -71,7 +71,7 @@ def test_channel_no_download_frequency(test_client, test_session, test_directory # Update the Channel with a frequency. new_download = {'urls': [simple_channel.url], 'frequency': 10, 'downloader': ChannelDownloader.name} request, response = test_client.post('/api/download', json=new_download) - assert response.status_code == HTTPStatus.NO_CONTENT, response.json + assert response.status_code == HTTPStatus.CREATED, response.json # Download is scheduled. Download is related to Channel. assert len(download_manager.get_downloads(test_session)) == 1 @@ -410,46 +410,47 @@ async def test_get_channel_videos_pagination(test_session, simple_channel, video @pytest.mark.asyncio -async def test_create_channel_download_api(async_client, test_session, simple_channel, tag_factory): - tag = await tag_factory() +async def test_channel_download_id(async_client, test_session, tag_factory, simple_channel, test_download_manager, + test_downloader): + tag = await tag_factory() - # Create Download. - body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}} - request, response = await async_client.post( - f'/api/videos/channels/{simple_channel.id}/download', json=body) - assert response.status_code == HTTPStatus.NO_CONTENT, response.content - download: Download = test_session.query(Download).one() - assert download.url == 'https://example.com/1' and not download.settings.get('tag_names') - assert download.settings.get('title_include') is None + # A recurring Download can be displayed on a Channel's page. + body = dict( + urls=['https://example.com/channel1'], + tag_names=[tag.name], + downloader=test_downloader.name, + settings=dict(channel_id=simple_channel.id), + frequency=120, + ) + request, response = await async_client.post(f'/api/download', json=body) + assert response.status_code == HTTPStatus.CREATED + download = test_session.query(Download).one() + assert download.url == 'https://example.com/channel1' + assert download.channel_id == simple_channel.id - # Create another Download. Previous Downloads is untouched. - body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}} - request, response = await async_client.post( - f'/api/videos/channels/{simple_channel.id}/download', json=body) - assert response.status_code == HTTPStatus.NO_CONTENT, response.content - download1, download2 = test_session.query(Download).order_by(Download.url).all() - assert download1.url == 'https://example.com/1' and not download1.settings.get('tag_names') - assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ] - test_session.flush(simple_channel) - assert len(simple_channel.downloads) == 2 - - # Change frequency of first Download. - body = {'url': 'https://example.com/1', 'frequency': 123, 'settings': {'title_include': 'foo,bar'}} - request, response = await async_client.put( - f'/api/videos/channels/{simple_channel.id}/download/{download1.id}', json=body) - assert response.status_code == HTTPStatus.NO_CONTENT, response.content - download1, download2 = test_session.query(Download).order_by(Download.url).all() - assert download1.url == 'https://example.com/1' - assert download1.settings == {'title_include': 'foo,bar'} - assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ] - test_session.flush(simple_channel) - assert len(simple_channel.downloads) == 2 - - # Deleting Download does not delete Channel. - request, response = await async_client.delete(f'/api/download/{download1.id}') + # A Channel relationship can be removed from a Download. + body = dict( + urls=['https://example.com/channel1'], + tag_names=[tag.name], + downloader=test_downloader.name, + settings=dict(), + frequency=240, + ) + request, response = await async_client.put(f'/api/download/{download.id}', json=body) assert response.status_code == HTTPStatus.NO_CONTENT - assert test_session.query(Download).count() == 1 - assert test_session.query(Channel).count() == 1 + download = test_session.query(Download).one() + assert download.channel_id is None + assert download.frequency == 240 + + # A once-Download cannot be associated with a Channel. + body = dict( + urls=['https://example.com/channel1'], + tag_names=[tag.name], + downloader=test_downloader.name, + settings=dict(channel_id=simple_channel.id), + ) + request, response = await async_client.put(f'/api/download/{download.id}', json=body) + assert response.status_code == HTTPStatus.BAD_REQUEST @pytest.mark.asyncio diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index d51f5582..843c9e16 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -151,7 +151,7 @@ def test_api_download(test_session, test_client, test_directory): content = dict(urls=['https://example.com/video1', ], downloader='video', destination='dest', settings=dict(excluded_urls='example.com')) request, response = test_client.post('/api/download', content=json.dumps(content)) - assert response.status_code == HTTPStatus.NO_CONTENT + assert response.status_code == HTTPStatus.CREATED download = test_session.query(Download).one() assert download.url == 'https://example.com/video1' @@ -167,15 +167,14 @@ async def test_api_download_video_tags(test_session, async_client, tag_factory): tag1 = await tag_factory() tag2 = await tag_factory() - content = dict(urls=['https://example.com/video1', ], downloader='video', - settings={'tag_names': [tag1.name, tag2.name]}) + content = dict(urls=['https://example.com/video1', ], downloader='video', tag_names=[tag1.name, tag2.name]) request, response = await async_client.post('/api/download', content=json.dumps(content)) - assert response.status_code == HTTPStatus.NO_CONTENT + assert response.status_code == HTTPStatus.CREATED download = test_session.query(Download).one() assert download.url == 'https://example.com/video1' assert download.downloader == 'video' - assert download.settings['tag_names'] == [tag1.name, tag2.name] + assert download.tag_names == [tag1.name, tag2.name] @pytest.mark.asyncio diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 744bbcca..547ada21 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -453,7 +453,7 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac @pytest.mark.asyncio async def test_channel_download_crud(test_session, async_client, assert_downloads, tag_factory): - """Test creatin""" + """Test creating more complex Channel Download.""" one, two = await tag_factory(), await tag_factory() download = { 'urls': ['https://www.youtube.com/@wrolpi/videos'], @@ -476,10 +476,11 @@ async def test_channel_download_crud(test_session, async_client, assert_download } request, response = await async_client.post('/api/download', json=download) - assert response.status == HTTPStatus.NO_CONTENT + assert response.status == HTTPStatus.CREATED, request.body download = test_session.query(Download).one() assert download.url == 'https://www.youtube.com/@wrolpi/videos' + assert download.destination is None assert download.tag_names == ['one', 'two'] assert download.downloader == 'video_channel' assert download.sub_downloader == 'video' diff --git a/modules/zim/schema.py b/modules/zim/schema.py index 046a382b..d6042d2a 100644 --- a/modules/zim/schema.py +++ b/modules/zim/schema.py @@ -5,7 +5,7 @@ from typing import Optional, List, Dict @dataclass class ZimSearchRequest: search_str: Optional[str] = None - tag_names: List[str] = field(default_factory=lambda: list()) + tag_names: List[str] = field(default_factory=list) offset: Optional[int] = 0 limit: Optional[int] = 10 @@ -98,5 +98,5 @@ class OutdatedZims: @dataclass class SearchEstimateRequest: search_str: Optional[str] = None - tag_names: List[str] = field(default_factory=lambda: list()) + tag_names: List[str] = field(default_factory=list) any_tag: bool = False diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 77388864..b8e800ef 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -584,6 +584,10 @@ class DownloadManager: if not frequency or not isinstance(frequency, int): raise ValueError('Recurring download must have a frequency!') + from wrolpi.scrape_downloader import ScrapeHTMLDownloader + if downloader_name == ScrapeHTMLDownloader.name and frequency: + raise InvalidDownload(f'Cannot schedule recurring download for {downloader_name=}') + download, = self.create_downloads([url, ], session=session, downloader_name=downloader_name, reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name, destination=destination, tag_names=tag_names, settings=settings) @@ -601,17 +605,22 @@ class DownloadManager: @optional_session def update_download(self, id_: int, url: str, downloader: str, destination: str | pathlib.Path = None, tag_names: List[str] = None, - sub_downloader: str | None = None, + sub_downloader: str | None = None, frequency: int = None, settings: Dict = None, session: Session = None) -> Download: download = self.find_download(session=session, id_=id_) + if settings and settings.get('channel_id') and not frequency: + raise InvalidDownload(f'A once-download cannot be associated with a Channel') download.url = url download.downloader = downloader + download.frequency = frequency # Preserve existing settings, unless new settings are provided. download.settings = settings if settings is not None else download.settings # Use provided params even if empty. download.destination = destination or None download.tag_names = tag_names or None download.sub_downloader = sub_downloader or None + # Remove Channel relationship, if necessary. + download.channel_id = (settings or dict()).get('channel_id') return download @wrol_mode_check diff --git a/wrolpi/files/test/test_scrape_downloader.py b/wrolpi/files/test/test_scrape_downloader.py index fddefe4b..3976d817 100644 --- a/wrolpi/files/test/test_scrape_downloader.py +++ b/wrolpi/files/test/test_scrape_downloader.py @@ -1,7 +1,10 @@ import asyncio +from http import HTTPStatus import mock +import pytest +from wrolpi.dates import Seconds from wrolpi.downloader import DownloadResult from wrolpi.files.downloader import FileDownloader from wrolpi.scrape_downloader import ScrapeHTMLDownloader @@ -76,3 +79,18 @@ async def test_scrape_html_downloader_html(test_directory, test_session, test_do 'https://example.com/dir', 'https://example.com/dir/other.html', ]) + + +@pytest.mark.asyncio +async def test_scrape_html_downloader_api(async_client, test_session, test_download_manager): + """Scrape Download cannot be recurring.""" + body = dict( + urls=['https://example.com'], + destination='uploads', + downloader='scrape_html', + sub_downloader='file', + frequency=Seconds.week, + tag_names=[], + settings=dict(depth=1, max_pages=1, suffix='.mp4')) + request, response = await async_client.post('/api/download', json=body) + assert response.status == HTTPStatus.BAD_REQUEST diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index f7d6604a..4fa6a349 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -255,13 +255,15 @@ def valid_regex(_: Request, body: schema.RegexRequest): @api_bp.post('/download') -@openapi.description('Download all the URLs that are provided.') -@validate(schema.DownloadRequest) +@openapi.definition( + description='Download all the URLs that are provided.', + body=schema.DownloadRequest, + validate=True, +) @wrol_mode_check async def post_download(_: Request, body: schema.DownloadRequest): - downloader = download_manager.find_downloader_by_name(body.downloader) - if not downloader: - raise InvalidDownload(f'Cannot find downloader with name {body.downloader}') + # Raises an InvalidDownload if the Downloader cannot be found. + download_manager.find_downloader_by_name(body.downloader) kwargs = dict(downloader_name=body.downloader, sub_downloader_name=body.sub_downloader, reset_attempts=True, @@ -275,17 +277,20 @@ async def post_download(_: Request, body: schema.DownloadRequest): # Downloads are disabled, warn the user. Events.send_downloads_disabled('Download created. But, downloads are disabled.') - return response.empty() + return response.empty(status=HTTPStatus.CREATED) @api_bp.put('/download/') -@openapi.description('Update properties of the Download') -@validate(schema.DownloadRequest) +@openapi.definition( + description='Update properties of the Download', + body=schema.DownloadRequest, + validate=True, +) @wrol_mode_check async def put_download(_: Request, download_id: int, body: schema.DownloadRequest): - downloader = download_manager.find_downloader_by_name(body.downloader) - if not downloader: - raise ValidationError(f'Cannot find downloader with name {body.downloader}') + # Raises an InvalidDownload if the Downloader cannot be found. + download_manager.find_downloader_by_name(body.downloader) + if len(body.urls) != 1: raise ValidationError('Only one URL can be specified when updating a Download') @@ -295,6 +300,7 @@ async def put_download(_: Request, download_id: int, body: schema.DownloadReques url=body.urls[0], downloader=body.downloader, destination=body.destination, + frequency=body.frequency, tag_names=body.tag_names, sub_downloader=body.sub_downloader, settings=body.settings, diff --git a/wrolpi/schema.py b/wrolpi/schema.py index f206cd3b..0a5a75ec 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -80,8 +80,8 @@ class EchoResponse: @dataclass class DownloadSettings: - channel_id: int = None - channel_tag_name: List[str] = field(default_factory=lambda: list()) + channel_id: Optional[int] = None + channel_tag_name: List[str] = field(default_factory=list) depth: Optional[int] = None download_metadata_only: bool = False download_order: Optional[str] = None @@ -90,16 +90,13 @@ class DownloadSettings: maximum_duration: Optional[int] = None minimum_duration: Optional[int] = None suffix: Optional[str] = None - tag_names: List[str] = field(default_factory=lambda: list()) - title_exclude: str = None - title_include: str = None + title_exclude: Optional[str] = None + title_include: Optional[str] = None video_count_limit: Optional[int] = None - video_format: str = None - video_resolutions: List[str] = field(default_factory=lambda: list()) + video_format: Optional[str] = None + video_resolutions: List[str] = field(default_factory=list) def __post_init__(self): - self.tag_names = [i.strip() for i in self.tag_names] if self.tag_names else [] - if self.excluded_urls and self.excluded_urls.endswith(','): raise ValidationError('Excluded urls cannot end with ,') if self.excluded_urls and self.excluded_urls.startswith(','): @@ -128,7 +125,7 @@ class DownloadRequest: urls: List[str] downloader: str destination: Optional[str] = None - tag_names: List[str] = field(default_factory=list) + tag_names: Optional[List[str]] = field(default_factory=list) frequency: Optional[int] = None sub_downloader: Optional[str] = None settings: Optional[dict] = field(default_factory=dict) @@ -146,6 +143,8 @@ class DownloadRequest: destination = get_relative_to_media_directory(destination) self.destination = str(destination) + self.tag_names = self.tag_names or [] + # Validate settings contents. Remove empty values. settings = self.settings or dict() self.settings = {k: v for k, v in DownloadSettings(**settings).__dict__.items() if v not in ([], None)} @@ -226,8 +225,8 @@ class SearchSuggestionsRequest: @dataclass class SearchFileEstimateRequest: search_str: Optional[str] = None - mimetypes: List[str] = field(default_factory=lambda: list()) - tag_names: List[str] = field(default_factory=lambda: list()) + mimetypes: List[str] = field(default_factory=list) + tag_names: List[str] = field(default_factory=list) months: Optional[List[int]] = None from_year: Optional[int] = None to_year: Optional[int] = None @@ -240,7 +239,7 @@ class SearchFileEstimateRequest: @dataclass class SearchOtherEstimateRequest: - tag_names: List[str] = field(default_factory=lambda: list()) + tag_names: List[str] = field(default_factory=list) @dataclass diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index d391bea3..d60dfa27 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -382,7 +382,7 @@ async def test_crud_download(async_client, test_session, test_download_manager, settings=dict(excluded_urls='example.org,something'), ) request, response = await async_client.post('/api/download', content=json.dumps(body)) - assert response.status_code == HTTPStatus.NO_CONTENT, response.body + assert response.status_code == HTTPStatus.CREATED, response.body download: Download = test_session.query(Download).one() assert download.id diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index dd2b571c..fb1ef349 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -430,42 +430,46 @@ async def test_download_crud(test_session, async_client, test_download_manager_c with mock.patch('wrolpi.downloader.DownloadManager.dispatch_downloads', dispatch_downloads): # Create a single recurring Download. - content = dict( + body = dict( urls=['https://example.com/1', ], frequency=1_000, downloader='archive', + tag_names=None, # tag_names can be a None. ) - request, response = await async_client.post('/api/download', content=json.dumps(content)) - assert response.status_code == HTTPStatus.NO_CONTENT + request, response = await async_client.post('/api/download', content=json.dumps(body)) + assert response.status_code == HTTPStatus.CREATED assert {i.url for i in test_session.query(Download)} == {'https://example.com/1', } # Create two once-downloads. - content = dict( + body = dict( urls=['https://example.com/2', 'https://example.com/3'], downloader='archive', + # tag_names=None, # tag_names can be missing. ) - request, response = await async_client.post('/api/download', json=content) - assert response.status_code == HTTPStatus.NO_CONTENT + request, response = await async_client.post('/api/download', json=body) + assert response.status_code == HTTPStatus.CREATED assert {i.url for i in test_session.query(Download)} == {f'https://example.com/{i}' for i in range(1, 4)} download1, download2, download3 = test_session.query(Download).order_by(Download.url).all() assert not download1.settings - expected_download1 = dict(download1.__json__()).copy() + expected_download = dict(download1.__json__()).copy() # Update a download. - content = dict( + body = dict( urls=['https://example.com/1'], downloader='archive', - settings={'tag_names': [tag1.name, tag2.name]}, + frequency=1_000, + tag_names=[tag1.name, tag2.name], ) - expected_download1.update({'url': 'https://example.com/1', 'settings': {'tag_names': [tag1.name, tag2.name]}}) - request, response = await async_client.put(f'/api/download/{download1.id}', json=content) + expected_download.update({'url': 'https://example.com/1', 'tag_names': [tag1.name, tag2.name]}) + request, response = await async_client.put(f'/api/download/{download1.id}', json=body) assert response.status_code == HTTPStatus.NO_CONTENT test_session.flush([download2]) - download2 = test_session.query(Download).filter_by(id=expected_download1['id']).first() - assert download2.__json__() == expected_download1 + download2 = test_session.query(Download).filter_by(id=expected_download['id']).first() + assert download2.__json__() == expected_download + assert download2.tag_names == [tag1.name, tag2.name] def test_get_downloaders(test_client): @@ -514,33 +518,6 @@ async def test_restart_download(test_session, async_client, test_download_manage assert download.is_deferred, download.status -@pytest.mark.asyncio -async def test_put_download(test_session, async_client, test_download_manager, test_downloader, channel_factory): - """A Download can be updated using a PUT request.""" - channel = channel_factory() - download = test_download_manager.create_download('https://example.com', test_downloader.name) - assert not download.settings - - body = dict( - downloader=test_downloader.name, - urls=['https://example.com/new-url'], # URL can be changed. - settings=dict(channel_id=channel.id), - ) - request, response = await async_client.put(f'/api/download/{download.id}', json=body) - assert response.status_code == HTTPStatus.NO_CONTENT - assert test_session.query(Download).count() == 1, 'Download should have been updated' - download = test_session.query(Download).one() - assert download.url == 'https://example.com/new-url', 'Download URL was not changed' - assert download.settings == dict(channel_id=channel.id), 'Download settings were not changed' - - # Settings can be cleared by sending `None` - body['settings'] = None - request, response = await async_client.put(f'/api/download/{download.id}', json=body) - assert response.status_code == HTTPStatus.NO_CONTENT - download = test_session.query(Download).one() - assert download.settings == dict(), 'Download settings were not changed' - - def test_get_global_statistics(test_session, test_client): request, response = test_client.get('/api/statistics') assert response.json['global_statistics']['db_size'] > 1