From 0f232fd533d3d7812dec729fce1c707ab53ac4f1 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 15 Aug 2024 19:32:01 -0600 Subject: [PATCH] Moving FileGroups instead of refreshing files after they have been moved. --- app/src/Events.js | 8 + app/src/api.js | 701 +++++++++++++++--------- app/src/components/Archive.js | 7 - app/src/components/Channels.js | 80 +-- app/src/components/Common.js | 6 +- app/src/components/Download.js | 19 +- app/src/components/Files.js | 6 + app/src/components/Inventory.js | 38 +- app/src/components/Search.js | 92 +++- app/src/components/Videos.js | 2 + app/src/components/admin/Downloads.js | 13 +- app/src/components/admin/Settings.js | 48 +- app/src/hooks/customHooks.js | 29 +- modules/archive/__init__.py | 17 +- modules/archive/conftest.py | 2 +- modules/archive/lib.py | 3 +- modules/archive/models.py | 4 +- modules/archive/test/test_api.py | 2 +- modules/videos/api.py | 20 + modules/videos/channel/api.py | 40 +- modules/videos/channel/lib.py | 63 ++- modules/videos/channel/test/test_api.py | 106 +++- modules/videos/channel/test/test_lib.py | 21 + modules/videos/common.py | 37 +- modules/videos/conftest.py | 12 +- modules/videos/downloader.py | 121 ++-- modules/videos/lib.py | 90 +-- modules/videos/models.py | 121 ++-- modules/videos/schema.py | 7 + modules/videos/test/test_api.py | 107 +++- modules/videos/test/test_common.py | 43 +- modules/videos/test/test_downloader.py | 2 +- modules/videos/test/test_models.py | 2 +- modules/videos/test/test_video.py | 26 +- modules/videos/video/lib.py | 5 +- modules/videos/video/test/test_api.py | 6 +- modules/zim/conftest.py | 8 +- modules/zim/lib.py | 15 +- modules/zim/test/test_api.py | 44 +- modules/zim/test/test_lib.py | 19 +- wrolpi/common.py | 77 ++- wrolpi/conftest.py | 32 +- wrolpi/contexts.py | 1 + wrolpi/downloader.py | 11 +- wrolpi/events.py | 13 +- wrolpi/files/api.py | 20 +- wrolpi/files/lib.py | 366 +++++++++---- wrolpi/files/models.py | 59 +- wrolpi/files/test/test_api.py | 31 +- wrolpi/files/test/test_ebooks.py | 2 +- wrolpi/files/test/test_lib.py | 76 ++- wrolpi/files/test/test_models.py | 21 + wrolpi/root_api.py | 34 +- wrolpi/schema.py | 5 + wrolpi/tags.py | 87 ++- wrolpi/test/test_common.py | 15 + wrolpi/test/test_downloader.py | 2 +- wrolpi/test/test_root_api.py | 32 +- wrolpi/test/test_tags.py | 66 ++- 59 files changed, 1902 insertions(+), 1040 deletions(-) create mode 100644 wrolpi/files/test/test_models.py diff --git a/app/src/Events.js b/app/src/Events.js index 2814b2d4..bfb1ec7a 100644 --- a/app/src/Events.js +++ b/app/src/Events.js @@ -75,6 +75,14 @@ function handleEvents(events) { eventToast('Successful tag', message, 'success'); } + if (event === 'file_move_completed') { + eventToast('Successful Move', message, 'success'); + } + + if (event === 'file_move_failed') { + eventToast('Move Failed', message, 'error'); + } + if (subject) { newestEvents[subject] = dt; } diff --git a/app/src/api.js b/app/src/api.js index c18b22fd..c75fa449 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -30,7 +30,7 @@ async function apiCall(url, method, body, ms = 60_000) { try { // await the response or error. - let response = await timeoutPromise(ms, promise); + const response = await timeoutPromise(ms, promise); if (200 <= response.status && response.status < 300) { // Request was successful. return response; @@ -38,7 +38,7 @@ async function apiCall(url, method, body, ms = 60_000) { // Request encountered an error. let copy = response.clone(); const content = await copy.json(); - console.debug('API error response json', content); + console.error('API error response json', content); const code = content['code']; if (response.status === 403 && code === 'WROL_MODE_ENABLED') { toast({ @@ -73,6 +73,16 @@ let apiPatch = async (url, body) => { return await apiCall(url, 'PATCH', body) }; +async function getErrorMessage(response, fallbackMessage) { + try { + const json = await response.clone().json(); + return json.error || json.message || fallbackMessage; + } catch (error) { + // Server did not response with JSON. Return default message. + return fallbackMessage; + } +} + export async function updateChannel(id, channel) { channel = emptyToNull(channel); return await apiPut(`${VIDEOS_API}/channels/${id}`, channel); @@ -86,29 +96,34 @@ export async function createChannel(channel) { export async function deleteChannel(channelId) { const response = await apiDelete(`${VIDEOS_API}/channels/${channelId}`); if (response.status !== 204) { - const content = await response.json(); - console.error(content); - throw Error(content); + const message = getErrorMessage(response, 'Failed to delete channel.'); + toast({ + type: 'error', + title: 'Unexpected server response', + description: message, + time: 5000, + }); } return response; } export async function getChannels() { - let response = await apiGet(`${VIDEOS_API}/channels`); + const response = await apiGet(`${VIDEOS_API}/channels`); if (response.status === 200) { return (await response.json())['channels']; } else { + const message = getErrorMessage(response, 'Failed to get channels.'); toast({ type: 'error', title: 'Unexpected server response', - description: 'Failed to get channels. See server logs.', + description: message, time: 5000, }); } } export async function getChannel(id) { - let response = await apiGet(`${VIDEOS_API}/channels/${id}`); + const response = await apiGet(`${VIDEOS_API}/channels/${id}`); return (await response.json())['channel']; } @@ -121,21 +136,30 @@ export async function tagChannel(channelId, tagName, directory) { } const response = await apiPost(`${VIDEOS_API}/channels/${channelId}/tag`, body); if (!response.ok) { + const message = await getErrorMessage(response, 'Failed to tag channel. See server logs.'); toast({ type: 'error', - title: 'Unexpected server response', - description: 'Failed to tag channel. See server logs.', + title: 'Tagging Channel Failed', + description: message, time: 5000, }); } } export async function tagChannelInfo(channelId, tagName) { - const body = {tag_name: tagName} - const response = await apiPost(`${VIDEOS_API}/channels/${channelId}/tag_info`, body); + channelId = channelId ? parseInt(channelId) : null; + const body = {channel_id: channelId, tag_name: tagName}; + const response = await apiPost(`${VIDEOS_API}/tag_info`, body); if (response.ok) { return (await response.json()).videos_destination; } + const message = getErrorMessage(response, 'Failed to get channel tag info.'); + toast({ + type: 'error', + title: 'Getting Channel Tag Info Failed', + description: message, + time: 5000, + }); } export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline) { @@ -159,15 +183,16 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by console.debug('searching videos', body); - let response = await apiPost(`${VIDEOS_API}/search`, body); + const response = await apiPost(`${VIDEOS_API}/search`, body); if (response.status === 200) { let data = await response.json(); return [data['file_groups'], data['totals']['file_groups']]; } else { + const message = getErrorMessage(response, 'Failed to search videos.'); toast({ type: 'error', - title: 'Unexpected server response', - description: 'Failed to search videos. See server logs.', + title: 'Searching Videos failed', + description: message, time: 5000, }); return [[], 0]; @@ -175,7 +200,7 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by } export async function getVideo(video_id) { - let response = await apiGet(`${VIDEOS_API}/video/${video_id}`); + const response = await apiGet(`${VIDEOS_API}/video/${video_id}`); let data = await response.json(); return [data['file_group'], data['prev'], data['next']]; } @@ -191,11 +216,15 @@ export async function deleteVideos(videoIds) { } console.info(`Deleting Videos: ${videoIds}`); const i = videoIds.join(','); - let response = await apiDelete(`${VIDEOS_API}/video/${i}`); + const response = await apiDelete(`${VIDEOS_API}/video/${i}`); if (response.status !== 204) { - const content = response.json(); - console.error(content); - throw Error(content); + const message = getErrorMessage(response, 'Failed to delete videos.'); + toast({ + type: 'error', + title: 'Deleting Videos failed', + description: message, + time: 5000, + }); } } @@ -217,64 +246,96 @@ export async function downloadVideoMetadata(videoUrl, destination) { export async function getDirectories(search_str) { let form_data = {'search_str': search_str || null}; - let response = await apiPost(`${API_URI}/files/directories`, form_data); - if (response.status === 200) { + const response = await apiPost(`${API_URI}/files/directories`, form_data); + if (response.ok) { return await response.json(); } return []; } export async function getStatus() { - let response = await apiGet(`${API_URI}/status`); + const response = await apiGet(`${API_URI}/status`); if (response.status === 200) { return await response.json(); } else { + const message = getErrorMessage(response, 'Could not get server status.'); toast({ - type: 'error', title: 'Unexpected server response', description: 'Could not get server status', time: 5000, + type: 'error', + title: 'Fetching Status Failed', + description: message, + time: 5000, }); } } export async function getSettings() { - let response = await apiGet(`${API_URI}/settings`); - if (response.status === 200) { + const response = await apiGet(`${API_URI}/settings`); + if (response.ok) { return await response.json(); } else { + const message = getErrorMessage(response, 'Could not get settings.'); toast({ - type: 'error', title: 'Unexpected server response', description: 'Could not get settings', time: 5000, + type: 'error', + title: 'Fetching Settings Failed', + description: message, + time: 5000, }); } } export async function saveSettings(settings) { - return await apiPatch(`${API_URI}/settings`, settings); + const response = await apiPatch(`${API_URI}/settings`, settings); + if (!response.ok) { + const message = getErrorMessage(response, 'Could not save settings.'); + toast({ + type: 'error', + title: 'Saving Settings Failed', + description: message, + time: 5000, + }); + } + return response } export async function getDownloads() { - let response = await apiGet(`${API_URI}/download`); - if (response.status === 200) { + const response = await apiGet(`${API_URI}/download`); + if (response.ok) { return await response.json(); } else { + const message = getErrorMessage(response, 'Could not get downloads.'); toast({ - type: 'error', title: 'Unexpected server response', description: 'Could not get downloads', time: 5000, + type: 'error', + title: 'Getting Downloads Failed', + description: message, + time: 5000, }); } + return response } export async function validateRegex(regex) { - let response = await apiPost(`${API_URI}/valid_regex`, {regex}); - return (await response.json())['valid']; + const response = await apiPost(`${API_URI}/valid_regex`, {regex}); + try { + return (await response.json())['valid']; + } catch (e) { + return false; + } } export async function getVideosStatistics() { - let response = await apiGet(`${VIDEOS_API}/statistics`); + const response = await apiGet(`${VIDEOS_API}/statistics`); if (response.status === 200) { return (await response.json())['statistics']; } else { + const message = getErrorMessage(response, 'Could not get statistics.'); toast({ - type: 'error', title: 'Unexpected server response', description: 'Could not get statistics', time: 5000, + type: 'error', + title: 'Getting Statistics Failed', + description: message, + time: 5000, }); } + return response } export async function createChannelDownload(channelId, url, frequency, title_include, title_exclude, tag_names) { @@ -288,7 +349,7 @@ export async function createChannelDownload(channelId, url, frequency, title_inc }, } url = `${VIDEOS_API}/channels/${channelId}/download`; - let response = await apiPost(url, body); + const response = await apiPost(url, body); if (!response.ok) { const json = await response.json(); if (json['code'] === 'INVALID_DOWNLOAD') { @@ -299,10 +360,11 @@ export async function createChannelDownload(channelId, url, frequency, title_inc time: 5000, }); } else { + const message = getErrorMessage(response, 'Could not create Channel download.'); toast({ type: 'error', - title: 'Failed to Download!', - description: 'Failed to create download. See server logs.', + title: 'Creating Channel Download Failed', + description: message, time: 5000, }); } @@ -321,7 +383,7 @@ export async function updateChannelDownload(channelId, downloadId, url, frequenc }, } url = `${VIDEOS_API}/channels/${channelId}/download/${downloadId}`; - let response = await apiPut(url, body); + const response = await apiPut(url, body); if (!response.ok) { const json = await response.json(); if (json['code'] === 'INVALID_DOWNLOAD') { @@ -332,10 +394,11 @@ export async function updateChannelDownload(channelId, downloadId, url, frequenc time: 5000, }); } else { + const message = getErrorMessage(response, 'Could not update Channel download.'); toast({ type: 'error', - title: 'Failed to Download!', - description: 'Failed to update download. See server logs.', + title: 'Updating Channel Download Failed', + description: message, time: 5000, }); } @@ -345,107 +408,185 @@ export async function updateChannelDownload(channelId, downloadId, url, frequenc export async function refreshChannel(channelId) { let url = `${VIDEOS_API}/channels/refresh/${channelId}`; - return await fetch(url, {method: 'POST'}); + const response = await apiPost(url); + if (!response.ok) { + const message = getErrorMessage(response, "Failed to refresh this channel's directory"); + toast({ + type: 'error', + title: 'Failed to refresh', + description: message, + time: 5000, + }) + } + return response; } export async function encryptOTP(otp, plaintext) { let body = {otp, plaintext}; - let response = await apiPost(`${OTP_API}/encrypt_otp`, body); - if (response.status === 200) { + const response = await apiPost(`${OTP_API}/encrypt_otp`, body); + if (response.ok) { return await response.json(); } else { + const message = getErrorMessage(response, 'Failed to encrypt OTP. See server logs.'); toast({ - type: 'error', title: 'Error!', description: 'Failed to encrypt OTP', time: 5000, - }); + type: 'error', + title: 'Failed to encrypt OTP', + description: message, + time: 5000, + }) } } export async function decryptOTP(otp, ciphertext) { let body = {otp, ciphertext}; - let response = await apiPost(`${OTP_API}/decrypt_otp`, body); - if (response.status === 200) { + const response = await apiPost(`${OTP_API}/decrypt_otp`, body); + if (response.ok) { return await response.json(); } else { + const message = getErrorMessage(response, 'Failed to decrypt OTP. See server logs.'); toast({ - type: 'error', title: 'Error!', description: 'Failed to decrypt OTP', time: 5000, - }); + type: 'error', + title: 'Failed to decrypt OTP', + description: message, + time: 5000, + }) } } export async function getCategories() { - let response = await apiGet(`${API_URI}/inventory/categories`); - if (response.status === 200) { + const response = await apiGet(`${API_URI}/inventory/categories`); + if (response.ok) { return (await response.json())['categories']; } else { + const message = getErrorMessage(response, 'Could not get categories'); toast({ - type: 'error', title: 'Unexpected server response', description: 'Could not get categories', time: 5000, + type: 'error', title: 'Unexpected server response', description: message, time: 5000, }); } } export async function getBrands() { - let response = await apiGet(`${API_URI}/inventory/brands`); - if (response.status === 200) { + const response = await apiGet(`${API_URI}/inventory/brands`); + if (response.ok) { return (await response.json())['brands']; } else { + const message = getErrorMessage(response, 'Could not get brands'); toast({ - type: 'error', title: 'Unexpected server response', description: 'Could not get brands', time: 5000, + type: 'error', title: 'Unexpected server response', description: message, time: 5000, }); } } export async function getInventories() { - let response = await apiGet(`${API_URI}/inventory`); - if (response.status === 200) { + const response = await apiGet(`${API_URI}/inventory`); + if (response.ok) { return (await response.json())['inventories']; } else { + const message = getErrorMessage(response, 'Could not get inventories'); toast({ - type: 'error', title: 'Unexpected server response', description: 'Could not get inventories', time: 5000, + type: 'error', title: 'Unexpected server response', description: message, time: 5000, }); } } export async function getInventory(inventoryId) { - let response = await apiGet(`${API_URI}/inventory/${inventoryId}`); - if (response.status === 200) { + const response = await apiGet(`${API_URI}/inventory/${inventoryId}`); + if (response.ok) { return await response.json(); } else { + const message = getErrorMessage(response, 'Could not get inventory'); toast({ - type: 'error', title: 'Unexpected server response', description: 'Could not get inventory', time: 5000, + type: 'error', title: 'Unexpected server response', description: message, time: 5000, }); } } export async function saveInventory(inventory) { - return await apiPost(`${API_URI}/inventory`, inventory); + const response = await apiPost(`${API_URI}/inventory`, inventory); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to save inventory'); + toast({ + type: 'error', title: 'Unexpected server response', description: message, time: 5000, + }); + } + return response } export async function updateInventory(inventoryId, inventory) { delete inventory['id']; - return await apiPut(`${API_URI}/inventory/${inventoryId}`, inventory); + const response = await apiPut(`${API_URI}/inventory/${inventoryId}`, inventory); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to update inventory'); + toast({ + type: 'error', title: 'Unexpected server response', description: message, time: 5000, + }); + } + return response } export async function deleteInventory(inventoryId) { - return await apiDelete(`${API_URI}/inventory/${inventoryId}`); + const response = await apiDelete(`${API_URI}/inventory/${inventoryId}`); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to delete inventory'); + toast({ + type: 'error', title: 'Unexpected server response', description: message, time: 5000, + }); + } + return response } export async function getItems(inventoryId) { - let response = await apiGet(`${API_URI}/inventory/${inventoryId}/item`); - return await response.json(); + const response = await apiGet(`${API_URI}/inventory/${inventoryId}/item`); + if (response.ok) { + return await response.json(); + } else { + const message = getErrorMessage(response, 'Failed to get items'); + toast({ + type: 'error', title: 'Unexpected server response', description: message, time: 5000, + }); + } + return response } export async function saveItem(inventoryId, item) { - return await apiPost(`${API_URI}/inventory/${inventoryId}/item`, item); + const response = await apiPost(`${API_URI}/inventory/${inventoryId}/item`, item); + if (response.ok) { + return await response.json(); + } else { + const message = getErrorMessage(response, 'Failed to save item'); + toast({ + type: 'error', title: 'Unexpected server response', description: message, time: 5000, + }); + } + return response } export async function updateItem(itemId, item) { item = emptyToNull(item); - return await apiPut(`${API_URI}/inventory/item/${itemId}`, item); + const response = await apiPut(`${API_URI}/inventory/item/${itemId}`, item); + if (response.ok) { + return await response.json(); + } else { + const message = getErrorMessage(response, 'Failed to update item'); + toast({ + type: 'error', title: 'Unexpected server response', description: message, time: 5000, + }); + } + return response } export async function deleteItems(itemIds) { let i = itemIds.join(','); - await apiDelete(`${API_URI}/inventory/item/${i}`); + const response = await apiDelete(`${API_URI}/inventory/item/${i}`); + if (response.ok) { + return await response.json(); + } else { + const message = getErrorMessage(response, 'Failed to delete items'); + toast({ + type: 'error', title: 'Unexpected server response', description: message, time: 5000, + }); + } + return response } export async function deleteArchives(archiveIds) { @@ -460,11 +601,14 @@ export async function deleteArchives(archiveIds) { console.log(`Deleting Archives: ${archiveIds}`); let i = archiveIds.join(','); const response = await apiDelete(`${API_URI}/archive/${i}`); - if (response.status !== 204) { - const content = await response.json(); - console.debug(content); - throw Error(content['error']); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to delete archives'); + toast({ + type: 'error', title: 'Unexpected server response', description: message, time: 5000, + }); + throw Error('Failed to delete archives'); } + return response } export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline) { @@ -489,15 +633,16 @@ export async function searchArchives(offset, limit, domain, searchStr, order, ta } console.debug('searching archives', body); - let response = await apiPost(`${ARCHIVES_API}/search`, body); - if (response.status === 200) { + const response = await apiPost(`${ARCHIVES_API}/search`, body); + if (response.ok) { let data = await response.json(); return [data['file_groups'], data['totals']['file_groups']]; } else { + const message = getErrorMessage(response, 'Cannot search archives. See server logs.'); toast({ type: 'error', title: 'Unable to search archives', - description: 'Cannot search archives. See server logs.', + description: message, time: 5000, }); return [[], 0]; @@ -505,15 +650,16 @@ export async function searchArchives(offset, limit, domain, searchStr, order, ta } export async function fetchDomains() { - let response = await apiGet(`${ARCHIVES_API}/domains`); - if (response.status === 200) { + const response = await apiGet(`${ARCHIVES_API}/domains`); + if (response.ok) { let data = await response.json(); return [data['domains'], data['totals']['domains']]; } else { + const message = getErrorMessage(response, 'Unable to fetch Domains. See server logs.'); toast({ type: 'error', title: 'Domains Error', - description: 'Unable to fetch Domains. See server logs.', + description: message, time: 5000, }); } @@ -521,17 +667,18 @@ export async function fetchDomains() { export async function getArchive(archiveId) { const response = await apiGet(`${ARCHIVES_API}/${archiveId}`); - if (response.status === 200) { + if (response.ok) { const data = await response.json(); return [data['file_group'], data['history']]; } else { + const message = getErrorMessage(response, 'Unable to get Archive. See server logs.'); toast({ - type: 'error', title: 'Archive Error', description: 'Unable to get Archive. See server logs.', time: 5000, + type: 'error', title: 'Archive Error', description: message, time: 5000, }); } } -function getDownloadSettings(excludedURLs, depth, suffix, tagNames, downloadMetadataOnly, destination, max_pages) { +function calculateDownloadSettings(excludedURLs, depth, suffix, tagNames, downloadMetadataOnly, destination, max_pages) { let settings = {}; if (excludedURLs) settings['excluded_urls'] = excludedURLs; if (depth) settings['depth'] = depth; @@ -566,7 +713,7 @@ export async function postDownload( throw new Error('downloader is required, but was not provided'); } - let settings = getDownloadSettings(excludedURLs, depth, suffix, tagNames, downloadMetadataOnly, destination, + let settings = calculateDownloadSettings(excludedURLs, depth, suffix, tagNames, downloadMetadataOnly, destination, max_pages); let body = { urls: urls, @@ -576,6 +723,12 @@ export async function postDownload( settings: settings, }; const response = await apiPost(`${API_URI}/download`, body); + if (!response.ok) { + const message = getErrorMessage(response, 'Unable to create Download. See server logs.'); + toast({ + type: 'error', title: 'Download Error', description: message, time: 5000, + }); + } return response; } @@ -597,7 +750,7 @@ export async function putDownload( throw new Error('downloader is required, but was not provided'); } - let settings = getDownloadSettings(excludedURLs) + let settings = calculateDownloadSettings(excludedURLs) let body = { urls: urls, downloader: downloader, @@ -606,31 +759,63 @@ export async function putDownload( settings: settings, }; const response = await apiPut(`${API_URI}/download/${download_id}`, body); + if (!response.ok) { + const message = getErrorMessage(response, 'Unable to update Download. See server logs.'); + toast({ + type: 'error', title: 'Download Error', description: message, time: 5000, + }); + } return response; } export async function killDownload(download_id) { - let response = await apiPost(`${API_URI}/download/${download_id}/kill`); + const response = await apiPost(`${API_URI}/download/${download_id}/kill`); + if (!response.ok) { + const message = getErrorMessage(response, 'Unable to stop Download. See server logs.'); + toast({ + type: 'error', title: 'Download Error', description: message, time: 5000, + }); + } return response; } export async function restartDownload(download_id) { - return await apiPost(`${API_URI}/download/${download_id}/restart`); + const response = await apiPost(`${API_URI}/download/${download_id}/restart`); + if (!response.ok) { + const message = getErrorMessage(response, 'Unable to restart Download. See server logs.'); + toast({ + type: 'error', title: 'Download Error', description: message, time: 5000, + }); + } + return response } export async function killDownloads() { - let response = await apiPost(`${API_URI}/download/kill`); + const response = await apiPost(`${API_URI}/download/kill`); + if (!response.ok) { + const message = getErrorMessage(response, 'Unable to stop downloading. See server logs.'); + toast({ + type: 'error', title: 'Download Error', description: message, time: 5000, + }); + } return response; } export async function startDownloads() { - let response = await apiPost(`${API_URI}/download/enable`); + const response = await apiPost(`${API_URI}/download/enable`); + if (!response.ok) { + const message = getErrorMessage(response, 'Unable to start downloading. See server logs.'); + toast({ + type: 'error', title: 'Download Error', description: message, time: 5000, + }); + } return response; } export async function getDownloaders() { try { - let response = await apiGet(`${API_URI}/downloaders`); + const response = await apiGet(`${API_URI}/downloaders`); + // Not toasting because this will happen often. return await response.json(); } catch (e) { return {downloaders: []}; @@ -639,12 +824,13 @@ export async function getDownloaders() { export async function deleteDownload(downloadId) { try { - let response = await apiDelete(`${API_URI}/download/${downloadId}`); - if (response.status !== 204) { + const response = await apiDelete(`${API_URI}/download/${downloadId}`); + if (!response.ok) { + const message = getErrorMessage(response, 'Unable to delete the download. See server logs.'); toast({ type: 'error', - title: 'Unable to delete', - description: 'Unable to delete the download. See server logs.', + title: 'Download Error', + description: message, time: 5000, }); } @@ -679,15 +865,16 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta console.info('searching files', body); const response = await apiPost(`${API_URI}/files/search`, body); - if (response.status === 200) { + if (response.ok) { let data = await response.json(); let [file_groups, total] = [data['file_groups'], data['totals']['file_groups']]; return [file_groups, total]; } else { + const message = getErrorMessage(response, 'Cannot search files. See server logs.'); toast({ type: 'error', title: 'Unable to search files', - description: 'Cannot search files. See server logs.', + description: message, time: 5000, }); return [null, null]; @@ -705,19 +892,27 @@ export async function refreshFiles(paths) { console.info(`Refreshing all files`); response = await apiPost(`${API_URI}/files/refresh`); } + if (!response.ok) { + const message = getErrorMessage(response, 'Cannot refresh files. See server logs.'); + toast({ + type: 'error', + title: 'Files Error', + description: message, + time: 5000, + }); + } return response; } export async function makeDirectory(path) { const body = {path: path}; - try { - await apiPost(`${API_URI}/files/directory`, body); - } catch (e) { - console.error(e); + const response = await apiPost(`${API_URI}/files/directory`, body); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to create directory. See server logs.'); toast({ type: 'error', title: 'Unable to create directory', - description: 'Failed to create directory. See server logs.', + description: message, time: 5000, }); } @@ -726,49 +921,52 @@ export async function makeDirectory(path) { export async function getFiles(directories) { console.debug(`getFiles ${directories}`); let body = {directories: directories || []}; - let response = await apiPost(`${API_URI}/files`, body); + const response = await apiPost(`${API_URI}/files`, body); + // Not toasting because this will happen often. let {files} = await response.json(); return files; } export async function getFile(path) { let body = {file: path}; - let response = await apiPost(`${API_URI}/files/file`, body); + const response = await apiPost(`${API_URI}/files/file`, body); if (response.ok) { const {file} = await response.json(); return file; } else { - throw new Error('Failed to get file data'); + const message = getErrorMessage(response, 'Failed to get file data. See server logs.'); + toast({ + type: 'error', + title: 'Files Error', + description: message, + time: 5000, + }); } } export async function deleteFile(paths) { let body = {paths: paths}; const response = await apiPost(`${API_URI}/files/delete`, body); - if (response.status === 409) { - const content = await response.json(); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to delete file. See server logs.'); toast({ type: 'error', - title: 'Delete error', - description: content['message'], + title: 'Files Error', + description: message, time: 5000, }); } + return response } export async function fetchFilesProgress() { const response = await apiGet(`${API_URI}/files/refresh_progress`); - if (response.status === 200) { + if (response.ok) { const json = await response.json(); return json['progress']; } } -export async function getHotspotStatus() { - let response = await getSettings(); - return response['hotspot_status']; -} - export async function setHotspot(on) { let response; if (on) { @@ -776,7 +974,7 @@ export async function setHotspot(on) { } else { response = await apiPost(`${API_URI}/hotspot/off`); } - if (response.status === 204) { + if (response.ok) { return null; } else { const content = await response.json(); @@ -789,18 +987,17 @@ export async function setHotspot(on) { time: 5000, }); } else { + const message = getErrorMessage(response, 'Could not modify hotspot. See server logs.'); toast({ - type: 'error', title: 'Error!', description: 'Could not modify hotspot. See server logs.', time: 5000, + type: 'error', + title: 'Hotspot Error', + description: message, + time: 5000, }); } } } -export async function getThrottleStatus() { - let response = await getSettings(); - return response['throttle_status']; -} - export async function setThrottle(on) { let response; if (on) { @@ -808,7 +1005,7 @@ export async function setThrottle(on) { } else { response = await apiPost(`${API_URI}/throttle/off`); } - if (response.status === 204) { + if (response.ok) { return null; } else { const content = await response.json(); @@ -821,101 +1018,65 @@ export async function setThrottle(on) { time: 5000, }); } else { - toast({ - type: 'error', title: 'Error!', description: 'Could not modify throttle. See server logs.', time: 5000, - }); + const message = getErrorMessage(response, 'Could not modify throttle. See server logs.'); + toast({type: 'error', title: 'Throttle Error', description: message, time: 5000}); } } } export async function getMapImportStatus() { - let response = await apiGet(`${API_URI}/map/files`); - if (response.status === 200) { + const response = await apiGet(`${API_URI}/map/files`); + if (response.ok) { return await response.json(); } else { - toast({ - type: 'error', title: 'Unexpected server response', description: 'Could not get import status', time: 5000, - }); + const message = getErrorMessage(response, 'Could not get import status'); + toast({type: 'error', title: 'Map Error', description: message, time: 5000}); } } export async function importMapFiles(paths) { let body = {'files': paths}; - let response = await apiPost(`${API_URI}/map/import`, body); - if (response.status === 204) { - return null; - } else { - toast({ - type: 'error', title: 'Error!', description: 'Could not start import! See server logs.', time: 5000, - }); + const response = await apiPost(`${API_URI}/map/import`, body); + if (!response.ok) { + const message = getErrorMessage(response, 'Could not start import! See server logs.'); + toast({type: 'error', title: 'Map Error', description: message, time: 5000}); } } export async function clearCompletedDownloads() { - let response = await apiPost(`${API_URI}/download/clear_completed`); - if (response.status === 204) { - return null - } else { - toast({ - type: 'error', - title: 'Error!', - description: 'Could not clear completed downloads! See server logs.', - time: 5000, - }); - } -} - -export async function clearFailedDownloads() { - let response = await apiPost(`${API_URI}/download/clear_failed`); - if (response.status === 204) { - return null - } else { - toast({ - type: 'error', - title: 'Error!', - description: 'Could not clear failed downloads! See server logs.', - time: 5000, - }); + const response = await apiPost(`${API_URI}/download/clear_completed`); + if (!response.ok) { + const message = getErrorMessage(response, 'Could not clear completed downloads! See server logs.'); + toast({type: 'error', title: 'Downloads Error', description: message, time: 5000}); } + return response } export async function deleteOnceDownloads() { - let response = await apiPost(`${API_URI}/download/delete_once`); - if (response.status === 204) { - return null - } else { - toast({ - type: 'error', - title: 'Error!', - description: 'Could not delete once downloads! See server logs.', - time: 5000, - }); + const response = await apiPost(`${API_URI}/download/delete_once`); + if (!response.ok) { + const message = getErrorMessage(response, 'Could not delete once downloads! See server logs.'); + toast({type: 'error', title: 'Downloads Error', description: message, time: 5000}); } + return response } export async function retryOnceDownloads() { - let response = await apiPost(`${API_URI}/download/retry_once`); - if (response.status === 204) { - return null - } else { - toast({ - type: 'error', - title: 'Error!', - description: 'Could not retry once downloads! See server logs.', - time: 5000, - }); + const response = await apiPost(`${API_URI}/download/retry_once`); + if (!response.ok) { + const message = getErrorMessage(response, 'Could not retry once downloads! See server logs.'); + toast({type: 'error', title: 'Downloads Error', description: message, time: 5000}); } + return response } export async function getStatistics() { - let response = await apiGet(`${API_URI}/statistics`); - if (response.status === 200) { - const contents = await response.json(); - return contents; + const response = await apiGet(`${API_URI}/statistics`); + if (response.ok) { + return await response.json(); } else { - toast({ - type: 'error', title: 'Error!', description: 'Unable to get file statistics', time: 5000, - }) + const message = getErrorMessage(response, 'Unable to get file statistics'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } @@ -924,19 +1085,23 @@ export async function getEvents(after) { if (after) { uri = `${uri}?after=${encodeURIComponent(after)}` } - let response = await apiGet(uri); - if (response.status === 200) { + const response = await apiGet(uri); + if (response.ok) { return await response.json(); } + // Not toasting because this happens often. + return response } export async function getTags() { - let uri = `${API_URI}/tag`; - let response = await apiGet(uri); - if (response.status === 200) { + const uri = `${API_URI}/tag`; + const response = await apiGet(uri); + if (response.ok) { const body = await response.json(); return body['tags']; } + // Not toasting because this happens often. + return response } export async function tagFileGroup(fileGroup, name) { @@ -952,9 +1117,10 @@ export async function tagFileGroup(fileGroup, name) { } const uri = `${API_URI}/files/tag`; - let response = await apiPost(uri, body); - if (response.status !== 201) { - console.error('Failed to add tag'); + const response = await apiPost(uri, body); + if (!response.ok) { + const message = getErrorMessage(response, 'Unable to add tag'); + toast({type: 'error', title: 'Tag Error', description: message, time: 5000}); } } @@ -971,9 +1137,10 @@ export async function untagFileGroup(fileGroup, name) { } const uri = `${API_URI}/files/untag`; - let response = await apiPost(uri, body) - if (response.status !== 204) { - console.error('Failed to remove tag'); + const response = await apiPost(uri, body) + if (!response.ok) { + const message = getErrorMessage(response, 'Unable to untag'); + toast({type: 'error', title: 'Tag Error', description: message, time: 5000}); } } @@ -983,7 +1150,7 @@ export async function saveTag(name, color, id) { if (id) { uri = `${uri}/${id}`; } - let response = await apiPost(uri, body); + const response = await apiPost(uri, body); if (id && response.status === 200) { toast({ type: 'info', title: 'Saved tag', description: `Saved tag: ${name}`, time: 2000, @@ -993,28 +1160,26 @@ export async function saveTag(name, color, id) { type: 'info', title: 'Created new tag', description: `Created new tag: ${name}`, time: 2000, }); } else { - console.error('Failed to create new tag'); - toast({ - type: 'error', title: 'Error!', description: 'Unable to save tag', time: 5000, - }) + const message = getErrorMessage(response, 'Unable to save tag'); + toast({type: 'error', title: 'Tag Error', description: message, time: 5000}); } } export async function deleteTag(id, name) { const uri = `${API_URI}/tag/${id}`; - let response = await apiDelete(uri); + const response = await apiDelete(uri); if (response.status === 400) { const content = await response.json(); + const message = getErrorMessage(response, 'Cannot delete, Tag is used'); if (content['code'] === 'USED_TAG') { toast({ - type: 'error', title: 'Error!', description: content['message'], time: 5000, + type: 'error', title: 'Error!', description: message, time: 5000, }) } - } else if (response.status !== 204) { + } else if (!response.ok) { console.error('Failed to delete tag'); - toast({ - type: 'error', title: 'Error!', description: `Unable to delete tag: ${name}`, time: 5000, - }) + const message = getErrorMessage(response, `Unable to delete tag: ${name}`); + toast({type: 'error', title: 'Tag Error', description: message, time: 5000}); } } @@ -1022,11 +1187,13 @@ export async function fetchFile(path) { const uri = `${API_URI}/files/file`; const body = {file: path}; const response = await apiPost(uri, body); - if (response.status === 200) { + if (response.ok) { const content = await response.json(); return content['file']; } else { console.error('Unable to fetch file dict! See client logs.'); + const message = getErrorMessage(response, 'Unable to get File'); + toast({type: 'error', title: 'File Error', description: message, time: 5000}); } } @@ -1036,8 +1203,8 @@ export async function sendNotification(message, url) { if (response.status === 201) { toast({type: 'success', title: 'Shared', description: 'Your share was sent', time: 2000}); } else { - toast({type: 'error', title: 'Error', description: 'Your share failed to send!', time: 5000}); - console.error('Failed to share'); + const message = getErrorMessage(response, 'Your share failed to send!'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } @@ -1047,61 +1214,65 @@ export async function searchDirectories(name) { if (response.status === 204) { return []; } else if (response.status === 200) { - const content = await response.json(); - return content; + return await response.json(); } else { - toast({type: 'error', title: 'Error', description: 'Failed to search directories!', time: 5000}); + const message = getErrorMessage(response, 'Failed to search directories!'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } export async function renamePath(path, newName) { const body = {path, new_name: newName}; const response = await apiPost(`${API_URI}/files/rename`, body); - if (response.status !== 204) { - toast({type: 'error', title: 'Error', description: 'Failed to rename!', time: 5000}); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to rename!'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } export async function movePaths(destination, paths) { const body = {destination, paths}; const response = await apiPost(`${API_URI}/files/move`, body); - if (response.status !== 204) { - const content = await response.json(); - toast({type: 'error', title: 'Error', description: content['api_error'], time: 5000}); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to move!'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } export async function ignoreDirectory(directory) { const body = {path: directory}; const response = await apiPost(`${API_URI}/files/ignore_directory`, body); - if (response.status !== 200) { - toast({type: 'error', title: 'Error', description: 'Failed to ignore directory!', time: 5000}); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to ignore directory!'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } export async function unignoreDirectory(directory) { const body = {path: directory}; const response = await apiPost(`${API_URI}/files/unignore_directory`, body); - if (response.status !== 200) { - toast({type: 'error', title: 'Error', description: 'Failed to unignore directory!', time: 5000}); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to unignore directory!'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } export async function fetchZims() { const response = await apiGet(`${API_URI}/zim`); - if (response.status === 200) { + if (response.ok) { const content = await response.json(); return { zims: content['zims'], } } else { - toast({type: 'error', title: 'Error', description: 'Cannot fetch Zims', time: 5000}); + const message = getErrorMessage(response, 'Cannot fetch Zims'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } export async function fetchZimSubscriptions() { const response = await apiGet(`${API_URI}/zim/subscribe`); - if (response.status === 200) { + if (response.ok) { const content = await response.json(); return { subscriptions: content['subscriptions'], @@ -1109,31 +1280,8 @@ export async function fetchZimSubscriptions() { iso_639_codes: content['iso_639_codes'], } } else { - toast({type: 'error', title: 'Error', description: 'Cannot fetch Zim Subscriptions', time: 5000}); - } -} - -export async function searchZims(offset, limit, searchStr) { - offset = parseInt(offset || 0); - limit = parseInt(limit || DEFAULT_LIMIT); - let body = {offset, limit}; - if (searchStr) { - body['search_str'] = searchStr; - } - - console.debug('searching zims', body); - let response = await apiPost(`${ZIM_API}/search`, body); - if (response.status === 200) { - let data = await response.json(); - return [data['zims'],]; - } else { - toast({ - type: 'error', - title: 'Unable to search zims', - description: 'Cannot search zims. See server logs.', - time: 5000, - }); - return [[], 0]; + const message = getErrorMessage(response, 'Cannot fetch Zim Subscriptions'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } @@ -1143,8 +1291,8 @@ export async function searchZim(offset, limit, searchStr, zimId, activeTags) { let body = {offset, limit, search_str: searchStr, tag_names: activeTags || []}; console.debug(`Searching Zim ${zimId} for: ${searchStr} tags: ${activeTags}`); - let response = await apiPost(`${ZIM_API}/search/${zimId}`, body); - if (response.status === 200) { + const response = await apiPost(`${ZIM_API}/search/${zimId}`, body); + if (response.ok) { const content = await response.json(); const zim = content['zim']; const searchLength = zim['search'].length; @@ -1157,9 +1305,10 @@ export async function tagZimEntry(zim_id, zim_entry, name) { const body = {tag_name: name, zim_id: zim_id, zim_entry: zim_entry}; const uri = `${API_URI}/zim/tag`; - let response = await apiPost(uri, body); - if (response.status !== 201) { - console.error('Failed to add tag'); + const response = await apiPost(uri, body); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to add zim tag'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } @@ -1167,9 +1316,10 @@ export async function untagZimEntry(zim_id, zim_entry, name) { const body = {tag_name: name, zim_id: zim_id, zim_entry: zim_entry}; const uri = `${API_URI}/zim/untag`; - let response = await apiPost(uri, body); - if (response.status !== 201) { - console.error('Failed to add tag'); + const response = await apiPost(uri, body); + if (!response.ok) { + const message = getErrorMessage(response, 'Failed to untag zim'); + toast({type: 'error', title: 'Error', description: message, time: 5000}); } } @@ -1206,6 +1356,8 @@ export async function searchSuggestions(search_str) { channels: content.channels, domains: content.domains, } + } else { + console.error('Failed to get file search suggestions!'); } } @@ -1240,6 +1392,28 @@ export async function searchEstimateZims(search_str, tagNames) { } } +export async function searchEstimateOthers(tagNames) { + const body = {tag_names: tagNames}; + const response = await apiPost(`${API_URI}/search_other_estimates`, body); + if (response.ok) { + const content = await response.json(); + return { + others: content.others, + } + } +} + +export async function searchChannels(tagNames) { + const body = {tag_names: tagNames}; + const response = await apiPost(`${API_URI}/videos/channels/search`, body); + if (response.ok) { + const content = await response.json(); + return { + channels: content.channels, + } + } +} + export async function getOutdatedZims() { const response = await apiGet(`${API_URI}/zim/outdated`); if (response.status === 200) { @@ -1249,10 +1423,11 @@ export async function getOutdatedZims() { current: content['current'], } } else { + const message = getErrorMessage(response, 'Cannot fetch outdated Zims. See server logs.'); toast({ type: 'error', - title: 'Unable to fetch', - description: 'Cannot fetch outdated Zims. See server logs.', + title: 'Error', + description: message, time: 5000, }); } diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index 97ca7d54..2de33524 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -118,13 +118,6 @@ function ArchivePage() { description: 'Archive update has been scheduled.', time: 2000, }); - } else { - toast({ - type: 'error', - title: 'Archive Downloading', - description: 'Archive update failed.', - time: 2000, - }); } } diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index 345ec301..5eefbfca 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -1,5 +1,5 @@ import React, {useState} from "react"; -import {Dimmer, Dropdown, Grid, Label, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react"; +import {Dimmer, Dropdown, Grid, Input, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react"; import { createChannel, createChannelDownload, @@ -19,6 +19,7 @@ import { frequencyOptions, HelpHeader, humanFileSize, + humanNumber, RequiredAsterisk, SearchInput, secondsToFrequency, @@ -77,6 +78,10 @@ function ChannelStatistics({statistics}) { {secondsToFullDuration(statistics.length)} Total Duration + + {humanNumber(statistics.video_tags)} + Video Tags + } @@ -231,15 +236,7 @@ function ChannelPage({create, header}) { if (e) { e.preventDefault(); } - const response = await refreshChannel(channelId); - if (response.status !== 204) { - toast({ - type: 'error', - title: 'Failed to refresh', - description: "Failed to refresh this channel's directory", - time: 5000, - }) - } + await refreshChannel(channelId); } const handleDelete = async () => { @@ -295,19 +292,36 @@ function ChannelPage({create, header}) { > {channel.tag_name ? 'Modify Tag' : 'Add Tag'} - handleTagSelect(null)} - /> -
- - {newTagDirectory && } + + + + handleTagSelect(null)} + /> + + + + + + + + + + setNewTagDirectory(value)} + disabled={!moveToTagDirectory} + /> + + +
@@ -354,7 +368,6 @@ function ChannelPage({create, header}) { Directory changeValue('directory', value)} placeholder='videos/channel directory' @@ -756,16 +769,21 @@ export function ChannelsPage() { ; const headers = [ - {key: 'name', text: 'Name', sortBy: 'name', width: 8}, - {key: 'tag', text: 'Tag', sortBy: 'tag_name', width: 2}, - {key: 'video_count', text: 'Videos', sortBy: 'video_count', width: 2}, - {key: 'download_frequency', text: 'Download Frequency', sortBy: 'minimum_frequency', width: 2}, - {key: 'size', text: 'Size', sortBy: 'size', width: 2}, + {key: 'name', text: 'Name', sortBy: [i => i['name'].toLowerCase()], width: 8}, + {key: 'tag', text: 'Tag', sortBy: [i => i['tag_name'], i => i['name'].toLowerCase()], width: 2}, + {key: 'video_count', text: 'Videos', sortBy: [i => i['video_count'], i => i['name'].toLowerCase()], width: 2}, + { + key: 'download_frequency', + text: 'Download Frequency', + sortBy: [i => i['minimum_frequency'], i => i['name'].toLowerCase()], + width: 2 + }, + {key: 'size', text: 'Size', sortBy: [i => i['size'], i => i['name'].toLowerCase()], width: 2}, {key: 'manage', text: 'Manage', width: 2}, ]; const mobileHeaders = [ - {key: 'name', text: 'Name', sortBy: 'name'}, - {key: 'video_count', text: 'Videos', sortBy: 'video_count'}, + {key: 'name', text: 'Name', sortBy: [i => i['name'].toLowerCase()]}, + {key: 'video_count', text: 'Videos', sortBy: [i => i['video_count'], i => i['name'].toLowerCase()]}, {key: 'manage', text: 'Manage'}, ]; diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 752cc314..c5a7572a 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -64,6 +64,7 @@ export const DEFAULT_LIMIT = 20; export const NAME = process.env && process.env.REACT_APP_NAME ? process.env.REACT_APP_NAME : null; // Other services on a WROLPi. +export const FILES_MEDIA_URI = `https://${window.location.hostname}:${window.location.port}/media/`; export const MAP_VIEWER_URI = `https://${window.location.hostname}:8084`; export const ZIM_VIEWER_URI = `https://${window.location.hostname}:8085`; export const HELP_VIEWER_URI = `https://${window.location.hostname}:8086`; @@ -1281,7 +1282,10 @@ export function DirectoryInput({disabled, error, placeholder, setInput, value, r setDirectory(value); } - const {media_directory} = settings; + let {media_directory} = settings; + if (media_directory && !media_directory.endsWith('/')) { + media_directory = `${media_directory}/`; + } error = error || isDirectory && !isDir; return
diff --git a/app/src/components/Download.js b/app/src/components/Download.js index 73197413..1d052944 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -43,10 +43,8 @@ class Downloader extends React.Component { this.setState({pending: true, submitted: false}); try { let response = await postDownload(urls, downloader, null, null, null, destination, tagNames); - if (response.status === 204) { + if (response.ok) { this.setState({urls: '', pending: false, submitted: true}); - } else { - toast({type: 'error', title: 'Error', description: 'Failed to create download!'}); } } finally { this.setState({pending: false}); @@ -216,8 +214,7 @@ class ChannelDownload extends React.Component { if (response.status === 204) { this.setState({pending: false, disabled: false, success: true, url: '', ready: false}); } else { - const content = await response.json(); - let error = content.message || null; + let error = (await response.json()).message || null; this.setState({pending: false, disabled: false, success: false, error}); } } @@ -355,8 +352,7 @@ class RSSDownload extends ChannelDownload { if (response.status === 204) { this.setState({pending: false, disabled: false, success: true, url: '', ready: false}); } else { - const content = await response.json(); - let error = content.message || null; + let error = (await response.json()).message || null; this.setState({pending: false, disabled: false, success: false, error}); } } @@ -487,15 +483,8 @@ class ScrapeDownloader extends Downloader { suffix, max_pages, ); - if (response.status === 204) { + if (response.ok) { this.setState({urls: '', pending: false, submitted: true}); - } else { - const content = await response.json(); - let error = 'Failed to create download!'; - if (content && content['error']) { - error = content['error']; - } - toast({type: 'error', title: 'Error', description: error}); } } finally { this.setState({pending: false}); diff --git a/app/src/components/Files.js b/app/src/components/Files.js index 39a4c126..50eb551e 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -20,7 +20,9 @@ import { ErrorMessage, ExternalCardLink, FileIcon, + FILES_MEDIA_URI, findPosterPath, + HandPointMessage, humanFileSize, isoDatetimeToString, mimetypeColor, @@ -583,6 +585,10 @@ function FilesPage() { return <> + + +

You can also view your media directory at {FILES_MEDIA_URI}

+
; } diff --git a/app/src/components/Inventory.js b/app/src/components/Inventory.js index 634cfcb2..4ad400e7 100644 --- a/app/src/components/Inventory.js +++ b/app/src/components/Inventory.js @@ -504,35 +504,19 @@ class EditInventory extends React.Component { handleSubmit = async (inventory, closeCallback) => { let inventoryId = inventory.id; - delete inventory['id']; let response = await updateInventory(inventoryId, inventory); - - if (response.status === 204) { + if (response.ok) { await this.props.success(); closeCallback(); - } else { - toast({ - type: 'error', - header: 'Failed to save inventory!', - description: 'Server responded with an error.', - time: 5000, - }) } } handleDelete = async (inventory, closeCallback) => { let response = await deleteInventory(inventory.id); - if (response.status === 204) { + if (response.ok) { await this.props.success(); closeCallback(); - } else { - toast({ - type: 'error', - header: 'Failed to delete inventory!', - description: 'Server responded with an error.', - time: 5000, - }) } } @@ -554,16 +538,9 @@ class NewInventory extends React.Component { delete inventory.id; let response = await saveInventory(inventory); - if (response.status === 201) { + if (response.ok) { await this.props.success(); closeCallback(); - } else { - toast({ - type: 'error', - header: 'Failed to save inventory!', - description: 'Server responded with an error.', - time: 5000, - }) } } @@ -953,18 +930,11 @@ class InventoryAddList extends React.Component { expiration_date: this.state.expiration_date, }; let response = await saveItem(this.state.inventory.id, item); - if (response.status === 204) { + if (response.ok) { await this.clearInputs(); await this.fetchItems(); await this.fetchCategories(); await this.fetchBrands(); - } else { - toast({ - type: 'error', - header: 'Failed to save item!', - description: 'Server responded with an error.', - time: 5000, - }) } } diff --git a/app/src/components/Search.js b/app/src/components/Search.js index dd6dc00f..31f1db26 100644 --- a/app/src/components/Search.js +++ b/app/src/components/Search.js @@ -1,15 +1,24 @@ import React, {useState} from "react"; -import {Route, Routes, useNavigate} from "react-router-dom"; +import {Link, Route, Routes, useNavigate} from "react-router-dom"; import {FilesSearchView} from "./Files"; -import {useLatestRequest, usePages, useSearchDate, useSearchFilter} from "../hooks/customHooks"; +import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchFilter} from "../hooks/customHooks"; import {ZimSearchView} from "./Zim"; -import {searchEstimateFiles, searchEstimateZims, searchSuggestions} from "../api"; +import {searchEstimateFiles, searchEstimateOthers, searchEstimateZims, searchSuggestions} from "../api"; import {filterToMimetypes, fuzzyMatch, normalizeEstimate, SearchResultsInput, TabLinks} from "./Common"; import _ from "lodash"; import {TagsContext} from "../Tags"; -import {Button as SButton, Header as SHeader, Label} from "semantic-ui-react"; -import {Modal, ModalContent} from "./Theme"; -import {QueryContext} from "../contexts/contexts"; +import { + AccordionContent, + AccordionTitle, + Button as SButton, + Grid, + GridColumn, + GridRow, + Header as SHeader, + Label +} from "semantic-ui-react"; +import {Accordion, Header, Icon, Loader, Modal, ModalContent, Segment} from "./Theme"; +import {QueryContext, ThemeContext} from "../contexts/contexts"; const SUGGESTED_APPS = [ {location: '/admin', title: 'Downloads', description: 'View and control your downloads'}, @@ -128,6 +137,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) { const {data: filesData, sendRequest: sendFilesRequest, loading: filesLoading} = useLatestRequest(500); // Zims are slow, so they are separate. const {data: zimData, sendRequest: sendZimRequest, loading: zimLoading} = useLatestRequest(500); + const {data: otherData, sendRequest: sendOtherRequest, loading: otherLoading} = useLatestRequest(500); React.useEffect(() => { if (searchStr || (tagNames && tagNames.length > 0)) { @@ -139,6 +149,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) { } sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag)); sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames)); + sendOtherRequest(async () => await searchEstimateOthers(tagNames)); } }, [ searchStr, @@ -179,6 +190,14 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) { } }, [setSuggestions, zimData]); + React.useEffect(() => { + if (!_.isEmpty(otherData)) { + setSuggestions((prevState) => { + return {...prevState, otherEstimates: otherData.others} + }); + } + }, [setSuggestions, otherData]); + return {suggestions, loading: generalLoading || zimLoading || filesLoading} } @@ -269,6 +288,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag) } } + const otherSum = suggestions.otherEstimates ? _.sum(Object.values(suggestions.otherEstimates)) : 0; + const matchingApps = SUGGESTED_APPS.filter(i => i.title.toLowerCase().includes(lowerSearchStr) || fuzzyMatch(i.title.toLowerCase(), lowerSearchStr) @@ -283,6 +304,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag) setSuggestionsSums({ fileGroups: newSuggestions.fileGroups, zims: zimSum, + otherSum: otherSum, channels: newSuggestions.channels.length, domains: newSuggestions.domains.length, tags: matchingTags?.length, @@ -342,10 +364,12 @@ export function SearchView({suggestions, suggestionsSums, loading}) { const filesTabName = Files ; const zimsTabName = Zims ; + const othersTabName = Other ; const links = [ {text: filesTabName, to: '/search', key: 'filesSearch_', end: true}, {text: zimsTabName, to: '/search/zim', key: 'zimsSearch'}, + {text: othersTabName, to: '/search/other', key: 'othersSearch'}, ]; return @@ -353,6 +377,7 @@ export function SearchView({suggestions, suggestionsSums, loading}) { }/> }/> + }/> } @@ -403,3 +428,58 @@ export function SearchIconButton() { } + +function SearchChannelPreview({channel}) { + const {t} = React.useContext(ThemeContext); + + return + + + {channel.name} + + + +} + +function OtherSearchView({loading}) { + const {searchParams} = React.useContext(QueryContext); + const [activeIndex, setActiveIndex] = React.useState(0); + const activeTags = searchParams.getAll('tag'); + const {channels, loading: channelsLoading} = useSearchChannels(activeTags); + + const handleClick = (newIndex) => { + console.log('newIndex', newIndex, 'activeIndex', activeIndex); + setActiveIndex(activeIndex === newIndex ? null : newIndex); + } + + if (loading || channelsLoading) { + return + + + } + + const channelsAccordion = + handleClick(0)} + > +
+ + Channels + +
+
+ + + {!_.isEmpty(channels) ? + channels.map(i => ) + : 'No Channels'} + + +
; + + return + {channelsAccordion} + +} \ No newline at end of file diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 14e1955f..d2bbb5f6 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -243,6 +243,7 @@ function VideosStatistics() { {key: 'month', label: 'Downloads Past Month'}, {key: 'year', label: 'Downloads Past Year'}, {key: 'sum_duration', label: 'Total Duration'}, + {key: 'censored_videos', label: 'Censored Videos'}, ]; const commentsNames = [ {key: 'have_comments', label: 'Have Comments'}, @@ -255,6 +256,7 @@ function VideosStatistics() { ]; const channelNames = [ {key: 'channels', label: 'Channels'}, + {key: 'tagged_channels', label: 'Tagged Channels'}, ]; const buildSegment = (title, names, stats) => { diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js index 45afe812..ebf732f1 100644 --- a/app/src/components/admin/Downloads.js +++ b/app/src/components/admin/Downloads.js @@ -255,18 +255,7 @@ class RecurringDownloadRow extends React.Component { handleRestart = async () => { const {id} = this.props; try { - const response = await restartDownload(id); - if (response.status !== 204) { - throw Error('Unable to restart download'); - } - } catch (e) { - toast({ - type: 'error', - title: 'Error', - description: 'Unable to restart download', - time: 5000, - }) - throw e; + await restartDownload(id); } finally { if (this.props.fetchDownloads) { this.props.fetchDownloads(); diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index 22d0c37d..351de605 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -181,7 +181,7 @@ export class SettingsPage extends React.Component { qrOpen: false, ready: false, - archive_directory: null, + archive_destination: null, download_on_startup: null, download_timeout: null, hotspot_device: null, @@ -190,12 +190,12 @@ export class SettingsPage extends React.Component { hotspot_status: null, ignore_outdated_zims: null, log_level: null, - map_directory: null, + map_destination: null, navColor: null, throttle_on_startup: null, throttle_status: null, - videos_directory: null, - zims_directory: null, + videos_destination: null, + zims_destination: null, } this.handleSubmit = this.handleSubmit.bind(this); @@ -207,7 +207,7 @@ export class SettingsPage extends React.Component { const settings = await getSettings(); this.setState({ ready: true, - archive_directory: settings.archive_directory, + archive_destination: settings.archive_destination, disabled: settings.wrol_mode, download_on_startup: settings.download_on_startup, download_timeout: settings.download_timeout || '', @@ -217,12 +217,12 @@ export class SettingsPage extends React.Component { hotspot_status: settings.hotspot_status, ignore_outdated_zims: settings.ignore_outdated_zims, log_level: fromApiLogLevel(settings.log_level), - map_directory: settings.map_directory, + map_destination: settings.map_destination, navColor: settings.nav_color || 'violet', throttle_on_startup: settings.throttle_on_startup, throttle_status: settings.throttle_status, - videos_directory: settings.videos_directory, - zims_directory: settings.zims_directory, + videos_destination: settings.videos_destination, + zims_destination: settings.zims_destination, }, this.handleHotspotChange); } catch (e) { console.error(e); @@ -233,7 +233,7 @@ export class SettingsPage extends React.Component { async handleSubmit(callback) { this.setState({disabled: true, pending: true}); let settings = { - archive_directory: this.state.archive_directory, + archive_destination: this.state.archive_destination, download_on_startup: this.state.download_on_startup, download_timeout: this.state.download_timeout ? parseInt(this.state.download_timeout) : 0, hotspot_device: this.state.hotspot_device, @@ -241,11 +241,11 @@ export class SettingsPage extends React.Component { hotspot_ssid: this.state.hotspot_ssid, ignore_outdated_zims: this.state.ignore_outdated_zims, log_level: toApiLogLevel(this.state.log_level), - map_directory: this.state.map_directory, + map_destination: this.state.map_destination, nav_color: this.state.navColor, throttle_on_startup: this.state.throttle_on_startup, - videos_directory: this.state.videos_directory, - zims_directory: this.state.zims_directory, + videos_destination: this.state.videos_destination, + zims_destination: this.state.zims_destination, } try { const response = await saveSettings(settings); @@ -304,7 +304,7 @@ export class SettingsPage extends React.Component { } const { - archive_directory, + archive_destination, disabled, download_on_startup, download_timeout, @@ -314,13 +314,13 @@ export class SettingsPage extends React.Component { hotspot_ssid, ignore_outdated_zims, log_level, - map_directory, + map_destination, navColor, pending, qrCodeValue, throttle_on_startup, - videos_directory, - zims_directory, + videos_destination, + zims_destination, } = this.state; const qrButton = ; @@ -468,9 +468,9 @@ export class SettingsPage extends React.Component { this.handleInputChange(e, 'archive_directory', d.value)} + onChange={(e, d) => this.handleInputChange(e, 'archive_destination', d.value)} /> @@ -479,9 +479,9 @@ export class SettingsPage extends React.Component { this.handleInputChange(e, 'videos_directory', d.value)} + onChange={(e, d) => this.handleInputChange(e, 'videos_destination', d.value)} /> @@ -492,9 +492,9 @@ export class SettingsPage extends React.Component { this.handleInputChange(e, 'map_directory', d.value)} + onChange={(e, d) => this.handleInputChange(e, 'map_destination', d.value)} /> @@ -503,9 +503,9 @@ export class SettingsPage extends React.Component { this.handleInputChange(e, 'zims_directory', d.value)} + onChange={(e, d) => this.handleInputChange(e, 'zims_destination', d.value)} /> diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index c8368885..0f4c86ff 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -17,7 +17,7 @@ import { getStatus, getVideo, getVideosStatistics, - searchArchives, + searchArchives, searchChannels, searchDirectories, searchVideos, searchZim, @@ -1230,4 +1230,31 @@ export const useDockerized = () => { export const useCalcQuery = () => { const [calc, setCalc] = useOneQuery('calc'); return {calc, setCalc} +} + +export const useSearchChannels = (defaultTagNames) => { + const [tagNames, setTagNames] = useState(defaultTagNames || []); + const [channels, setChannels] = useState([]); + const [loading, setLoading] = useState(false); + + const localSearchChannels = async () => { + setLoading(true); + try { + const {channels: newChannels} = await searchChannels(tagNames); + setChannels(newChannels); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + localSearchChannels(); + }, [tagNames]); + + return { + tagNames, + setTagNames, + channels, + loading, + } } \ No newline at end of file diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index aa4a2bcd..29eeaa4b 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -49,17 +49,20 @@ class ArchiveDownloader(Downloader, ABC): singlefile, readability, screenshot = await self.do_archive(download) archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot) + archive_id = archive.id - if download.settings and (tag_names := download.settings.get('tag_names')): - for name in tag_names: - archive.add_tag(name) + with get_db_session() as session: + archive = Archive.find_by_id(archive_id, session) + if download.settings and (tag_names := download.settings.get('tag_names')): + for name in tag_names: + archive.add_tag(name, session) - if session := Session.object_session(archive): - session.commit() + if session := Session.object_session(archive): + session.commit() - logger.info(f'Successfully downloaded Archive {download.url} {archive}') + logger.info(f'Successfully downloaded Archive {download.url} {archive}') - return DownloadResult(success=True, location=f'/archive/{archive.id}') + return DownloadResult(success=True, location=f'/archive/{archive.id}') @optional_session def already_downloaded(self, *urls: List[str], session: Session = None) -> List: diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py index 4065cd8f..0898d8ee 100644 --- a/modules/archive/conftest.py +++ b/modules/archive/conftest.py @@ -82,7 +82,7 @@ def archive_factory(test_session, archive_directory, make_files_structure): archive.validate() for tag_name in tag_names: - archive.add_tag(tag_name) + archive.add_tag(tag_name, test_session) return archive diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 45992410..f67e6277 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -205,7 +205,8 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr archive.file_group.download_datetime = now() archive.url = url archive.domain = get_or_create_domain(session, url) - session.flush() + archive.flush() + archive.domain.flush() return archive diff --git a/modules/archive/models.py b/modules/archive/models.py index 05ea88a0..0e5a311f 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -315,8 +315,8 @@ class Archive(Base, ModelHelper): archive = model_archive(file_group, session) return archive - def add_tag(self, tag_id_or_name: int | str) -> TagFile: - return self.file_group.add_tag(tag_id_or_name) + def add_tag(self, tag_id_or_name: int | str, session: Session = None) -> TagFile: + return self.file_group.add_tag(tag_id_or_name, session) class Domain(Base, ModelHelper): diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py index db0e4139..0af2e71e 100644 --- a/modules/archive/test/test_api.py +++ b/modules/archive/test/test_api.py @@ -84,7 +84,7 @@ def test_archives_search(test_session, archive_directory, archive_factory, test_ @pytest.mark.asyncio async def test_search_archive_tags(test_session, test_async_client, archive_factory, tag_factory): """Tagged Archives can be searched.""" - tag = tag_factory() + tag = await tag_factory() archive_factory(domain='example.com', tag_names=[tag.name, ]) test_session.commit() diff --git a/modules/videos/api.py b/modules/videos/api.py index a1ed068a..21a64860 100644 --- a/modules/videos/api.py +++ b/modules/videos/api.py @@ -2,12 +2,15 @@ from http import HTTPStatus from sanic import Blueprint from sanic.request import Request +from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi.api_utils import json_response from wrolpi.common import logger from . import lib, schema from .channel.api import channel_bp +from .lib import format_videos_destination +from .models import Channel from .video.api import video_bp content_bp = Blueprint('VideoContent', '/api/videos') @@ -25,3 +28,20 @@ logger = logger.getChild(__name__) async def statistics(_: Request): ret = await lib.get_statistics() return json_response(ret, HTTPStatus.OK) + + +@content_bp.post('/tag_info') +@openapi.definition( + description='Get data about tagging a Channel', + body=schema.ChannelTagInfoRequest, +) +@validate(schema.ChannelTagInfoRequest) +async def channel_tag_info(_: Request, body: schema.ChannelTagInfoRequest): + from wrolpi.tags import Tag + channel = Channel.find_by_id(body.channel_id) if body.channel_id else None + channel_name = channel.name if channel else None + channel_url = channel.url if channel else None + tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None + videos_destination = format_videos_destination(channel_name, tag_name, channel_url) + ret = dict(videos_destination=videos_destination) + return json_response(ret) diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index 4bc5ade3..72b6db22 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -17,7 +17,7 @@ from wrolpi.vars import PYTEST from . import lib from .. import schema from ..errors import UnknownChannel -from ..lib import save_channels_config, format_videos_destination +from ..lib import save_channels_config from ..models import Channel channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels') @@ -78,9 +78,9 @@ def channel_post(_: Request, body: schema.ChannelPostRequest): @validate(schema.ChannelPutRequest) @run_after(save_channels_config) @wrol_mode_check -def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest): +async def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest): body.directory = pathlib.Path(body.directory) if body.directory else None - channel = lib.update_channel(data=body, channel_id=channel_id) + channel = await lib.update_channel(data=body, channel_id=channel_id) return response.raw('', HTTPStatus.NO_CONTENT, headers={'Location': f'/api/videos/channels/{channel.id}'}) @@ -138,32 +138,14 @@ async def channel_download(_: Request, channel_id: int, body: schema.ChannelDown return response.empty() -@channel_bp.post('//tag_info') -@openapi.definition( - description='Get Channel Tag Info', - body=schema.ChannelTagInfoRequest, -) -@validate(schema.ChannelTagInfoRequest) -@run_after(save_channels_config) -@wrol_mode_check -async def channel_tag_info(_: Request, channel_id: int, body: schema.ChannelTagInfoRequest): - from wrolpi.tags import Tag - channel_name = Channel.find_by_id(channel_id).name - tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None - videos_destination = format_videos_destination(channel_name, tag_name) - ret = dict(videos_destination=videos_destination) - return json_response(ret) - - @channel_bp.post('//tag') @openapi.definition( - description='Tag Channel', + description='Tag/untag a Channel with a single tag', body=schema.ChannelTagRequest, ) @openapi.response(HTTPStatus.NO_CONTENT) @openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) @validate(schema.ChannelTagRequest) -@run_after(save_channels_config) @wrol_mode_check async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagRequest): directory = None @@ -174,3 +156,17 @@ async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagReques await lib.tag_channel(body.tag_name, directory, channel_id) return response.raw('', HTTPStatus.NO_CONTENT) + + +@channel_bp.post('/search') +@openapi.definition( + description='Search Channels', + body=schema.ChannelSearchRequest, +) +@validate(schema.ChannelSearchRequest) +async def channel_search(_: Request, body: schema.ChannelSearchRequest): + channels = await lib.search_channels(body.tag_names) + ret = dict( + channels=channels, + ) + return json_response(ret) \ No newline at end of file diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index ed7ee5ee..fdca41fb 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -7,10 +7,11 @@ from sqlalchemy.orm import Session from wrolpi import flags from wrolpi.common import run_after, logger, \ - get_media_directory, wrol_mode_check + get_media_directory, wrol_mode_check, background_task from wrolpi.db import get_db_curs, optional_session, get_db_session from wrolpi.downloader import save_downloads_config, download_manager -from wrolpi.errors import UnknownDirectory, APIError, ValidationError, RefreshConflict +from wrolpi.errors import APIError, ValidationError, RefreshConflict +from wrolpi.vars import PYTEST from .. import schema from ..common import check_for_channel_conflicts from ..errors import UnknownChannel, ChannelDirectoryConflict @@ -115,22 +116,10 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No @run_after(save_channels_config) @optional_session -def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel: +async def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel: """Update a Channel's DB record""" channel = Channel.find_by_id(channel_id, session) - # Only update directory if it was empty - if data.directory and not channel.directory: - data.directory = get_media_directory() / data.directory - if not data.directory.is_dir(): - if data.mkdir: - data.directory.mkdir() - else: - raise UnknownDirectory() - elif data.directory: - # Keep the old directory because the user can't change a Channel's directory. - data.directory = channel.directory - # Verify that the URL/Name/directory aren't taken check_for_channel_conflicts( session, @@ -140,11 +129,27 @@ def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_ directory=data.directory, ) + data = data.__dict__ + old_directory = pathlib.Path(channel.directory) + new_directory = get_media_directory() / data.pop('directory') + # Apply the changes now that we've OK'd them - channel.update(data.__dict__) + channel.update(data) session.commit() + async def _(): + await channel.move_channel(new_directory, session, send_events=True) + + if new_directory != old_directory: + if not new_directory.is_dir(): + new_directory.mkdir(parents=True) + + if PYTEST: + await _() + else: + background_task(_()) + return channel @@ -252,7 +257,7 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f @wrol_mode_check @optional_session -async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id: int, session: Session = None): +async def tag_channel(tag_name: str | None, directory: pathlib.Path | None, channel_id: int, session: Session = None): """Add a Tag to a Channel, or remove a Tag from a Channel if no `tag_name` is provided. Move the Channel to the new directory, if provided.""" @@ -261,29 +266,39 @@ async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id: if directory and flags.refreshing.is_set(): raise RefreshConflict('Refusing to move channel while file refresh is in progress') - channel = Channel.find_by_id(channel_id, session=session) + channel = Channel.find_by_id(channel_id, session) if tag_name: - tag = Tag.find_by_name(tag_name, session=session) + tag = Tag.find_by_name(tag_name, session) channel.tag_id = tag.id channel.tag = tag else: channel.tag = channel.tag_id = None - session.flush([channel, ]) + channel.flush() + session.commit() # Do not move Channel files into a directory with files. - if directory and directory.is_dir() and next(directory.iterdir(), None): + if directory and directory != channel.directory and directory.is_dir() and next(directory.iterdir(), None): raise ChannelDirectoryConflict('Channel directory already exists and is not empty') # Only move Channel when requested. if directory: # Move to newly defined directory only if necessary. - logger.info(f'Tagging {channel} with {tag_name} moving to {directory} from {channel.directory}') directory.mkdir(parents=True, exist_ok=True) if channel.directory != directory: - await channel.move(directory, session) + coro = channel.move_channel(directory, session, send_events=True) + if PYTEST: + await coro + else: + background_task(coro) else: logger.info(f'Tagging {channel} with {tag_name}') - session.commit() + +@optional_session +async def search_channels(tag_names: List[str], session: Session) -> List[Channel]: + """Search Tagged Channels.""" + from wrolpi.tags import Tag + channels = session.query(Channel).join(Tag).filter(Tag.name.in_(tag_names)).all() + return channels diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index c615bb4f..1309d776 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -1,6 +1,7 @@ import asyncio import json import pathlib +import shutil import tempfile from datetime import timedelta from http import HTTPStatus @@ -96,7 +97,8 @@ def test_video_file_name(test_session, simple_video, test_client): assert resp.json['file_group']['video'].get('stem') == 'simple_video' -def test_channel_conflicts(test_client, test_session, test_directory): +@pytest.mark.asyncio +async def test_channel_conflicts(test_async_client, test_session, test_directory): channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name pathlib.Path(channel_directory).mkdir() new_channel = dict( @@ -105,11 +107,11 @@ def test_channel_conflicts(test_client, test_session, test_directory): url='https://example.com/channel1', ) - def _post_channel(channel): - return test_client.post('/api/videos/channels', content=json.dumps(channel)) + async def _post_channel(channel): + return await test_async_client.post('/api/videos/channels', content=json.dumps(channel)) # Create it - request, response = _post_channel(new_channel) + request, response = await _post_channel(new_channel) assert response.status_code == HTTPStatus.CREATED # Name is an exact match @@ -119,7 +121,7 @@ def test_channel_conflicts(test_client, test_session, test_directory): directory=channel_directory2, name='Example Channel 1', ) - request, response = _post_channel(new_channel) + request, response = await _post_channel(new_channel) assert response.status_code == HTTPStatus.BAD_REQUEST assert response.json == {'cause': {'code': 'CHANNEL_NAME_CONFLICT', 'error': 'The channel name is already taken.', @@ -133,7 +135,7 @@ def test_channel_conflicts(test_client, test_session, test_directory): directory=channel_directory, name='name is fine', ) - request, response = _post_channel(new_channel) + request, response = await _post_channel(new_channel) assert response.status_code == HTTPStatus.BAD_REQUEST assert response.json == {'cause': {'code': 'CHANNEL_DIRECTORY_CONFLICT', 'error': 'The directory is already used by another channel.', @@ -150,7 +152,7 @@ def test_channel_conflicts(test_client, test_session, test_directory): name='name is fine', url='https://example.com/channel1', ) - request, response = _post_channel(new_channel) + request, response = await _post_channel(new_channel) assert response.status_code == HTTPStatus.BAD_REQUEST assert response.json == {'cause': {'code': 'CHANNEL_URL_CONFLICT', 'error': 'The URL is already used by another channel.', @@ -384,7 +386,8 @@ def test_search_videos_channel(test_client, test_session, video_factory): dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id))) -def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search): +@pytest.mark.asyncio +async def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search): for i in range(50): video_factory(channel_id=simple_channel.id) @@ -406,8 +409,9 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto ] last_ids = [] for offset, video_count in tests: - _, response = assert_video_search(channel_id=simple_channel.id, order_by='published_datetime', offset=offset, - limit=20) + _, response = await assert_video_search(channel_id=simple_channel.id, order_by='published_datetime', + offset=offset, + limit=20) assert len(response.json['file_groups']) == video_count, 'Returned videos does not match' current_ids = [i['id'] for i in response.json['file_groups']] assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}' @@ -416,7 +420,7 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto @pytest.mark.asyncio async def test_create_channel_download_api(test_async_client, test_session, simple_channel, tag_factory): - tag = tag_factory() + tag = await tag_factory() # Create Download. body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}} @@ -465,7 +469,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan videos_directory = get_videos_directory() channel = channel_factory(name='Channel Name', download_frequency=120) channel_directory = channel.directory - tag = tag_factory('Tag Name') + tag = await tag_factory('Tag Name') v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id) # Create recurring download which uses the Channel's directory. download = download_manager.recurring_download('https://example.com/1', 60, test_downloader.name, @@ -532,7 +536,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved' assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved' - # Channel can be Tagged, but not move directories. + # Channel can be Tagged, without moving directories. body = dict(tag_name=tag.name) request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body) assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode() @@ -545,3 +549,79 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan d1, d2 = test_session.query(Download).all() assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved' assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved' + + +@pytest.mark.asyncio +async def test_move_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory, + video_factory, test_channels_config, test_download_manager, test_downloader): + """A Channel can be moved.""" + channel = channel_factory(name='Channel Name') + vid1 = video_factory(channel_id=channel.id) + vid2 = video_factory(channel_id=channel.id) + test_session.commit() + + assert str(channel.directory) == str(test_directory / 'videos/Channel Name') + assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/')) + assert str(vid2.video_path).startswith(str(test_directory / 'videos/Channel Name/')) + + body = dict( + name=channel.name, + directory=str(test_directory / 'videos/New Directory'), + ) + request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode() + channel = Channel.find_by_id(channel.id) + assert str(channel.directory) == str(test_directory / 'videos/New Directory') + assert str(vid1.video_path).startswith(str(test_directory / 'videos/New Directory/')) + assert str(vid2.video_path).startswith(str(test_directory / 'videos/New Directory/')) + + +@pytest.mark.asyncio +async def test_move_channel_after_terminal_move(test_async_client, test_session, test_directory, channel_factory, + tag_factory, video_factory, test_channels_config, test_download_manager, + test_downloader): + """A Channel can be moved in the UI after being moved in the Terminal.""" + channel = channel_factory(name='Channel Name') + vid1 = video_factory(channel_id=channel.id) + test_session.commit() + + assert str(channel.directory) == str(test_directory / 'videos/Channel Name') + assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/')) + + # mv 'videos/Channel Name' 'videos/New Directory' + shutil.move(channel.directory, test_directory / 'videos/New Directory') + + body = dict( + name=channel.name, + directory=str(test_directory / 'videos/New Directory'), + ) + request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode() + channel = Channel.find_by_id(channel.id) + vid1 = test_session.query(Video).one() + assert str(channel.directory) == str(test_directory / 'videos/New Directory') + assert str(vid1.video_path).startswith(str(test_directory / 'videos/New Directory/')) + + +@pytest.mark.asyncio +async def test_search_tagged_channels(test_async_client, test_session, channel_factory, tag_factory, + video_factory): + """Tagged Channels can be searched.""" + tag = await tag_factory() + test_session.commit() + + # Can search when empty. + body = dict(tag_names=[tag.name]) + request, response = await test_async_client.post('/api/videos/channels/search', json=body) + assert response.status_code == HTTPStatus.OK + assert len(response.json['channels']) == 0, 'No Channels should be tagged' + + channel1 = channel_factory(tag_name=tag.name) + channel2 = channel_factory() + test_session.commit() + + body = dict(tag_names=[tag.name]) + request, response = await test_async_client.post('/api/videos/channels/search', json=body) + assert response.status_code == HTTPStatus.OK + assert len(response.json['channels']) == 1, 'Only one Channel is tagged' + assert response.json['channels'][0]['id'] == channel1.id diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py index c9100110..3a36d021 100644 --- a/modules/videos/channel/test/test_lib.py +++ b/modules/videos/channel/test/test_lib.py @@ -1,6 +1,7 @@ from typing import List import pytest +import yaml from modules.videos import schema from modules.videos.channel import lib @@ -124,3 +125,23 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact await assert_search('foo', ['foo', 'foo bar', 'foobar']) await assert_search('bar', ['foobar', 'foo bar', 'bar'], True) + + +@pytest.mark.asyncio +async def test_tag_channel_existing(test_async_client, test_session, test_directory, channel_factory, tag_factory, + video_factory, test_channels_config): + """A Channel can already exist in a Tag's directory and still be tagged.""" + channel_directory = test_directory / 'videos/one/Channel Name' + channel = channel_factory(name='Channel Name', download_frequency=120, directory=channel_directory) + video_factory(channel_id=channel.id) + test_session.commit() + assert str(channel.directory).startswith(f'{test_directory}/videos/one/') + + tag = await tag_factory() + await lib.tag_channel(tag.name, channel_directory, channel.id, test_session) + + # Channel's tag is saved to the config. + save_channels_config(test_session) + with test_channels_config.open() as fh: + config = yaml.load(fh, Loader=yaml.Loader) + assert config['channels'][0]['tag_name'] == tag.name diff --git a/modules/videos/common.py b/modules/videos/common.py index 8a4307ef..9ba724f2 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -17,7 +17,7 @@ from sqlalchemy.orm import Session from wrolpi.captions import FFMPEG_BIN from wrolpi.cmd import FFPROBE_BIN -from wrolpi.common import logger, get_media_directory, get_wrolpi_config +from wrolpi.common import logger, get_media_directory from wrolpi.dates import seconds_to_timestamp from wrolpi.db import get_db_session, get_db_curs from wrolpi.vars import DEFAULT_FILE_PERMISSIONS @@ -204,7 +204,7 @@ def is_valid_poster(poster_path: Path) -> bool: async def ffprobe_json(video_path: Union[Path, str]) -> dict: """Extract video file metadata using ffprobe. - >>> ffprobe_json('video') + >>> ffprobe_json('video.mp4') { 'chapters': [], 'format': {}, @@ -270,39 +270,6 @@ def get_video_duration(video_path: Path) -> Optional[int]: return duration -def check_for_video_corruption(video_path: Path) -> bool: - """Uses ffprobe to check for specific ways a video file can be corrupt. - - Also attempts to screenshot the video near the end of it's duration, this check if the video is corrupted somewhere - in the middle.""" - video_path = Path(video_path) if not isinstance(video_path, Path) else video_path - - if not video_path.is_file(): - raise FileNotFoundError(f'{video_path} does not exist!') - if not FFPROBE_BIN: - raise SystemError('ffprobe is not installed!') - - # Just read the video using FFMPEG, this prints out useful information about the video. - cmd = (FFPROBE_BIN, str(video_path)) - try: - proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except subprocess.CalledProcessError as e: - logger.warning(f'FFPROBE failed to check for corruption with stderr: {e.stderr.decode()}') - return True # video is corrupt. - - messages = ( - b'Invalid NAL unit size', - b'Error splitting the input into NAL units', - b'Invalid data found when processing input', - ) - corrupt = False - for error in messages: - if error in proc.stderr: - logger.warning(f'Possible video corruption ({error.decode()}): {video_path}') - corrupt = True - return corrupt - - async def update_view_counts_and_censored(channel_id: int): """Update view_count for all Videos in a channel using its info_json file. Also sets FileGroup.censored if Video is no longer available on the Channel.""" diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 0e2f7240..6a3e71b1 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -40,14 +40,16 @@ def channel_factory(test_session, test_directory): def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None, directory: pathlib.Path = None, tag_name: str = None): name = name or str(uuid4()) - tag = Tag.get_by_name(tag_name) if tag_name else None + tag = Tag.find_by_name(tag_name) if tag_name else None channel = Channel( name=name, url=url or f'https://example.com/{name}', source_id=source_id, tag=tag, + tag_id=tag.id if tag else None, ) - channel.directory = directory or format_videos_destination(name, tag, channel.url) + tag_name = tag.name if tag else None + channel.directory = directory or format_videos_destination(name, tag_name, channel.url) channel.directory.mkdir(exist_ok=True, parents=True) test_session.add(channel) test_session.flush([channel]) @@ -242,12 +244,12 @@ def video_with_search_factory(test_session, test_directory, video_file_factory): @pytest.fixture -def assert_video_search(test_client): +def assert_video_search(test_async_client): """A fixture which performs a video search request against the API. If assert_* params are passed, the response is checked.""" - def _search_videos( + async def _search_videos( assert_total: int = None, assert_ids: List[int] = None, assert_paths: List[str] = None, @@ -272,7 +274,7 @@ def assert_video_search(test_client): if channel_id is not None: content['channel_id'] = channel_id - request, response = test_client.post('/api/videos/search', content=json.dumps(content)) + request, response = await test_async_client.post('/api/videos/search', content=json.dumps(content)) assert response.json assert response.status_code == HTTPStatus.OK diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 2979846f..c77b0eab 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -4,7 +4,7 @@ import os.path import pathlib import traceback from abc import ABC -from typing import Tuple, List, Dict +from typing import Tuple, List, Dict, Optional import yt_dlp.utils from sqlalchemy.orm import Session @@ -64,6 +64,7 @@ def extract_info(url: str, ydl: YoutubeDL = YDL, process=False) -> dict: def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str: """Get filename from YoutubeDL. Separated for testing.""" dir_name, file_name = os.path.split(ydl.prepare_filename(entry)) + file_name = escape_file_name(file_name) file_name = trim_file_name(file_name) return f'{dir_name}/{file_name}' @@ -241,8 +242,8 @@ class VideoDownloader(Downloader, ABC): # Video may have been downloaded previously, get its location for error reporting. with get_db_session() as session: - videos = session.query(Video).join(FileGroup).filter_by(url=url).all() - location = videos[0].location if videos else None + video_ = Video.get_by_url(url, session) + location = video_.location if video_ else None try: download.info_json = download.info_json or extract_info(url) @@ -257,62 +258,7 @@ class VideoDownloader(Downloader, ABC): if not download.info_json: raise ValueError(f'Cannot download video with no info_json.') - found_channel = None - - # Look for Channel using channel data first, fallback to uploader data. - channel_name = download.info_json.get('channel') or download.info_json.get('uploader') - channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id') - channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url') - channel = None - if channel_name or channel_source_id or channel_url: - # Try to find the channel via info_json from yt-dlp. - try: - channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name) - if channel: - found_channel = 'yt_dlp' - logger.debug(f'Found a channel with source_id {channel=}') - except UnknownChannel: - # Can't find a channel, use the no channel directory. - pass - - settings = download.settings or dict() - destination = settings.get('destination') - if not channel and destination: - # Destination may find Channel if not already found. - try: - channel = get_channel(directory=destination, return_dict=False) - found_channel = 'download_settings_directory' - logger.debug(f'Found a channel that shares the destination directory {channel=}') - except UnknownChannel: - # Destination must not be a channel. - pass - - local_channel_id = settings.get('channel_id') - channel_url = settings.get('channel_url') - if not channel and (local_channel_id or channel_url): - # Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download. - logger.info(f'Using download.settings to find channel') - try: - channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False) - logger.debug(f'Found channel with channel url {channel=}') - except UnknownChannel: - # We do not need a Channel if we have a destination directory. - if not destination: - raise - if channel: - found_channel = 'download_settings' - - channel_id = channel.id if channel else None - channel_directory = channel.directory if channel else None - - if found_channel == 'yt_dlp': - logger.debug(f'Found {channel} using yt_dlp') - elif found_channel == 'download_settings': - logger.info(f'Found {channel} using Download.settings') - elif found_channel == 'download_settings_directory': - logger.info(f'Found {channel} using Download.settings directory') - else: - logger.warning('Could not find channel') + channel, channel_directory, channel_id, destination, settings = await self._get_channel(download) if destination: # Download to the directory specified in the settings. @@ -480,6 +426,63 @@ class VideoDownloader(Downloader, ABC): ) return result + @staticmethod + async def _get_channel(download: Download) \ + -> Tuple[Optional[Channel], Optional[pathlib.Path], Optional[int], Optional[pathlib.Path], dict]: + found_channel = None + # Look for Channel using channel data first, fallback to uploader data. + channel_name = download.info_json.get('channel') or download.info_json.get('uploader') + channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id') + channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url') + channel = None + if channel_name or channel_source_id or channel_url: + # Try to find the channel via info_json from yt-dlp. + try: + channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name) + if channel: + found_channel = 'yt_dlp' + logger.debug(f'Found a channel with source_id {channel=}') + except UnknownChannel: + # Can't find a channel, use the no channel directory. + pass + settings = download.settings or dict() + destination = settings.get('destination') + if not channel and destination: + # Destination may find Channel if not already found. + try: + channel = get_channel(directory=destination, return_dict=False) + found_channel = 'download_settings_directory' + logger.debug(f'Found a channel that shares the destination directory {channel=}') + except UnknownChannel: + # Destination must not be a channel. + pass + channel_id = settings.get('channel_id') + channel_url = settings.get('channel_url') + if not channel and (channel_id or channel_url): + # Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download. + logger.info(f'Using download.settings to find channel') + try: + channel = get_channel(channel_id=channel_id, url=channel_url, return_dict=False) + logger.debug(f'Found channel with channel url {channel=}') + except UnknownChannel: + # We do not need a Channel if we have a destination directory. + if not destination: + raise + if channel: + found_channel = 'download_settings' + channel_id = channel.id if channel else None + channel_directory = channel.directory if channel else None + if found_channel == 'yt_dlp': + logger.debug(f'Found {channel} using yt_dlp') + elif found_channel == 'download_settings': + logger.info(f'Found {channel} using Download.settings') + elif found_channel == 'download_settings_directory': + logger.info(f'Found {channel} using Download.settings directory') + else: + logger.warning('Could not find channel') + + return channel, channel_directory, channel_id, destination, settings + @staticmethod def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]: """Get the full path of a video file from its URL.""" diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 265031d5..f79f15c8 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -3,6 +3,7 @@ import dataclasses import html import pathlib import re +import traceback from typing import Generator from typing import Tuple, Optional @@ -15,7 +16,7 @@ from modules.videos.models import Video from wrolpi import before_startup, dates, flags from wrolpi.captions import extract_captions from wrolpi.common import ConfigFile, limit_concurrent, get_wrolpi_config, extract_domain, \ - escape_file_name, get_media_directory + escape_file_name, get_media_directory, background_task from wrolpi.dates import Seconds, from_timestamp from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.downloader import Download @@ -379,6 +380,22 @@ def save_channels_config(session: Session = None): channel_import_logger = logger.getChild('channel_import') +async def fetch_channel_source_id(channel_id: int): + # If we can download from a channel, we must have its source_id. + try: + with get_db_session() as session: + from modules.videos.channel.lib import get_channel + channel = Channel.find_by_id(channel_id) + channel.source_id = channel.source_id or get_channel_source_id(channel.url) + if not channel.source_id: + channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}') + else: + session.commit() + save_channels_config(session) + except Exception as e: + logger.error(f'Failed to get Channel source id of id={channel_id}', exc_info=e) + + @before_startup @limit_concurrent(1) def import_channels_config(): @@ -387,13 +404,17 @@ def import_channels_config(): logger.warning('Skipping import_channels_config for this test') return + from .channel.lib import get_channel channel_import_logger.info('Importing channels config') try: config = get_channels_config() channels = config.channels channel_directories = [i['directory'] for i in channels] + if len(channel_directories) != len(set(channel_directories)): + logger.error('Refusing to import channels config because it contains duplicate directories!') + return + updated_channel_ids = set() - save_config = False with get_db_session(commit=True) as session: for data in channels: if isinstance(data, str): @@ -405,45 +426,37 @@ def import_channels_config(): for option in (i for i in REQUIRED_OPTIONS if i not in data): raise ConfigError(f'Channel "{directory}" is required to have "{option}"') - channel = session.query(Channel).filter( - or_( - Channel.directory == str(directory), - Channel.directory == str(data['directory']), - Channel.directory == str(directory.relative_to(get_media_directory())), - ), - ).one_or_none() + # Try to find Channel by directory because it is unique. + channel = Channel.get_by_path(directory, session) if not channel: - # Channel not yet in the DB, add it. - channel = Channel(directory=directory) - session.add(channel) - session.flush([channel, ]) - # TODO refresh the files in the channel. - logger.warning(f'Creating new Channel from config {channel}') - - # Only name and directory are required - channel.name = data['name'] - channel.directory = directory - - # A URL should not be an empty string - data['url'] = data['url'] or None + try: + # Try to find Channel using other attributes before creating new Channel. + channel = get_channel( + session, + source_id=data.get('source_id'), + url=data.get('url'), + directory=str(directory), + return_dict=False, + ) + except UnknownChannel: + # Channel not yet in the DB, add it. + channel = Channel(directory=directory) + session.add(channel) + channel.flush() + # TODO refresh the files in the channel. + logger.warning(f'Creating new Channel from config {channel}') # Copy existing channel data, update all values from the config. This is necessary to clear out # values not in the config. full_data = channel.dict() full_data.update(data) channel.update(full_data) + updated_channel_ids.add(channel.id) if not channel.source_id and channel.url and flags.have_internet.is_set(): # If we can download from a channel, we must have its source_id. - try: - channel.source_id = get_channel_source_id(channel.url) - save_config = True - if not channel.source_id: - channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}') - else: - session.commit() - except Exception as e: - logger.error(f'Failed to get Channel source id of {channel}', exc_info=e) + logger.info(f'Fetching channel source id for {channel}') + background_task(fetch_channel_source_id(channel.id)) channel_import_logger.debug(f'Updated {repr(channel.name)}' f' url={channel.url}' @@ -454,17 +467,13 @@ def import_channels_config(): with get_db_session(commit=True) as session: # Delete any Channels that were deleted from the config. for channel in session.query(Channel): - if str(channel.directory) not in channel_directories: + if channel.id not in updated_channel_ids: logger.warning(f'Deleting {channel} because it is not in the config.') channel.delete_with_videos() # Create any missing Downloads. Associated Downloads with any necessary Channels. link_channel_and_downloads() - if save_config: - # Information about the channel was fetched, store it. - save_channels_config() - except Exception as e: channel_import_logger.warning('Failed to load channels config!', exc_info=e) if PYTEST: @@ -544,7 +553,8 @@ async def get_statistics(): COUNT(v.id) FILTER ( WHERE v.have_comments = TRUE ) AS "have_comments", COUNT(v.id) FILTER ( WHERE v.have_comments = FALSE AND v.comments_failed = FALSE and fg.censored = false and fg.url is not null) AS "missing_comments", - COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments" + COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments", + COUNT(v.id) FILTER ( WHERE fg.censored = TRUE ) as "censored_videos" FROM video v LEFT JOIN file_group fg on v.file_group_id = fg.id @@ -583,9 +593,11 @@ async def get_statistics(): curs.execute(''' SELECT - COUNT(id) AS "channels" + COUNT(c.id) AS "channels", + COUNT(c.id) FILTER ( WHERE c.tag_id IS NOT NULL ) AS "tagged_channels" FROM - channel + channel c + LEFT JOIN public.tag t on t.id = c.tag_id ''') channel_stats = dict(curs.fetchone()) ret = dict(statistics=dict( diff --git a/modules/videos/models.py b/modules/videos/models.py index 1b9b598e..0442fe14 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -1,4 +1,5 @@ import json +import json import pathlib from typing import Optional, Dict, List @@ -7,11 +8,14 @@ from sqlalchemy.orm import relationship, Session, deferred from sqlalchemy.orm.collections import InstrumentedList from modules.videos.errors import UnknownVideo, UnknownChannel +from wrolpi import flags from wrolpi.captions import read_captions -from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file +from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file, \ + unique_by_predicate from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.downloader import Download, download_manager, save_downloads_config from wrolpi.errors import InvalidDownload +from wrolpi.events import Events from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, move from wrolpi.files.models import FileGroup from wrolpi.media_path import MediaPathType @@ -223,6 +227,7 @@ class Video(ModelHelper, Base): # Can't validate if there is no video file. logger.error(f'Unable to validate video {self.id} without primary file!') + logger.debug(f'Validating {self}') from .lib import validate_video try: validate_video(self, self.channel.generate_posters if self.channel else False) @@ -237,12 +242,14 @@ class Video(ModelHelper, Base): # self.file_group.d_text is handled in `validate_video`. if (comments := self.get_comments()) and len(comments) >= 5: + logger.debug(f'{self} has comments') self.have_comments = True # If this Video is in a Channel's directory, then it is part of that Channel. session: Session = Session.object_session(self) if session and (channel := Channel.get_by_path(self.video_path.parent, session)): self.channel = channel + logger.debug(f'{self} has Channel {channel}') @staticmethod def from_paths(session: Session, *paths: pathlib.Path) -> 'Video': @@ -533,7 +540,7 @@ class Channel(ModelHelper, Base): videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id') downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id') tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE')) - tag = relationship('Tag') + tag = relationship('Tag', primaryjoin='Channel.tag_id==Tag.id') def __repr__(self): return f'' @@ -565,12 +572,9 @@ class Channel(ModelHelper, Base): channel_id = self.id data = data.copy() - # Remove unused values. - data.pop('move_to_tag_directory', None) - # URL should not be empty string. - if 'url' in data: - data['url'] = data['url'] or None + url = data.pop('url') + self.url = url or None session: Session = Session.object_session(self) @@ -664,23 +668,19 @@ class Channel(ModelHelper, Base): with get_db_curs() as curs: stmt = ''' SELECT - SUM(size), - MAX(size), - COUNT(video.id), - SUM(fg.length) + SUM(size) AS "size", + MAX(size) AS "largest_video", + COUNT(video.id) AS "video_count", + SUM(fg.length) AS "length", + -- Videos may use multiple tags. + COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags" FROM video LEFT JOIN file_group fg on fg.id = video.file_group_id + LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id WHERE channel_id = %(id)s ''' curs.execute(stmt, dict(id=self.id)) - size, largest_video, video_count, length = curs.fetchone() - statistics = dict( - video_count=video_count, - size=size, - largest_video=largest_video, - length=length, - ) - return statistics + return dict(curs.fetchone()) async def refresh_files(self, send_events: bool = True): """Refresh all files within this Channel's directory. Mark this channel as refreshed.""" @@ -772,39 +772,72 @@ class Channel(ModelHelper, Base): channel = session.query(Channel).filter_by(url=url).one_or_none() return channel - async def move(self, directory: pathlib.Path, session: Session): + def format_directory(self, tag_name: str) -> pathlib.Path: + from modules.videos.lib import format_videos_destination + return format_videos_destination(self.name, tag_name, self.url) + + async def move_channel(self, directory: pathlib.Path, session: Session, send_events: bool = False): """Move the files of this Channel into a new directory.""" if not directory.is_dir(): raise FileNotFoundError(f'Destination directory does not exist: {directory}') - if self.directory and not next(self.directory.iterdir(), None): - raise FileExistsError(f'Refusing to move Channel when it has no files.') old_directory = self.directory self.directory = directory - session.flush([self, ]) - # Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's - # directory. - downloads = list(self.downloads) - downloads.extend(list( - session.query(Download).filter(Download.settings.contains({'destination': str(old_directory)})) - )) - for download in downloads: - # Get new copy of `settings` to avoid bug where it wouldn't update. - settings = download.settings.copy() if download.settings else dict() - if settings.get('destination'): - download.settings = {**settings, 'destination': str(directory)} + def change_download_destinations(from_directory: pathlib.Path, to_directory: pathlib.Path): + downloads = list(self.downloads) + downloads.extend(Download.get_all_by_destination(from_directory)) + downloads = unique_by_predicate(downloads, lambda i: i.id) + for download in downloads: + # Get new copy of `settings` to avoid bug where it wouldn't update. + settings = download.settings.copy() if download.settings else dict() + if settings.get('destination'): + logger.debug(f'Updating destination of {download}') + download.settings = {**settings, 'destination': str(to_directory)} + session.flush(downloads) - session.commit() + # Only one tag can be moved at a time. + with flags.refreshing: + # Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's + # directory. + change_download_destinations(old_directory, directory) - # Save configs before move, this is because move imports configs. - from modules.videos.lib import save_channels_config - await save_downloads_config(session) - save_channels_config(session) - save_tags_config(session) + session.commit() - # Move the contents of the Channel directory into the destination directory. - await move(directory, *list(old_directory.iterdir())) + # Save configs before move, this is because move imports configs. + from modules.videos.lib import save_channels_config + await save_downloads_config(session) + save_channels_config(session) + save_tags_config(session) - if old_directory.exists() and not next(old_directory.iterdir(), None): - old_directory.rmdir() + # Move the contents of the Channel directory into the destination directory. + logger.info(f'Moving {self} from {repr(str(old_directory))}') + + try: + if not old_directory.exists(): + # Old directory does not exist, maintainer must have moved the Channel manually. + if PYTEST: + await refresh_files([old_directory, directory]) + else: + background_task(refresh_files([old_directory, directory])) + if send_events: + Events.send_file_move_completed(f'Channel {repr(self.name)} was moved, but Tags were lost') + else: + # `move` also uses `flags.refreshing` + await move(directory, *list(old_directory.iterdir())) + if send_events: + Events.send_file_move_completed(f'Channel {repr(self.name)} was moved') + except Exception as e: + logger.error(f'Channel move failed! Reverting changes...', exc_info=e) + # Downloads must be moved back to the old directory. + change_download_destinations(directory, old_directory) + self.directory = old_directory + self.flush(session) + if send_events: + Events.send_file_move_failed(f'Moving Channel {self.name} has failed') + raise + finally: + session.commit() + if old_directory.exists() and not next(iter(old_directory.iterdir()), None): + # Old directory is empty, delete it. + old_directory.rmdir() diff --git a/modules/videos/schema.py b/modules/videos/schema.py index 4760149d..a21b1eee 100644 --- a/modules/videos/schema.py +++ b/modules/videos/schema.py @@ -49,6 +49,7 @@ class ChannelTagRequest: @dataclass class ChannelTagInfoRequest: + channel_id: Optional[int] = None tag_name: Optional[str] = None @@ -205,6 +206,7 @@ class VideoStatistics: @dataclass class ChannelStatistics: channels: int + tagged_channels: int @dataclass @@ -222,3 +224,8 @@ class CensoredVideoRequest: @dataclass class CensoredVideoResponse: videos: List[VideoWithChannel] + + +@dataclass +class ChannelSearchRequest: + tag_names: List[str] = field(default_factory=list) diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index fd82274d..105fec06 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -6,6 +6,7 @@ import pytest from modules.videos.models import Video, Channel from modules.videos.video import lib as video_lib +from wrolpi.common import get_wrolpi_config from wrolpi.db import get_db_curs from wrolpi.downloader import Download from wrolpi.files.lib import refresh_files @@ -160,14 +161,15 @@ def test_api_download(test_session, test_client, test_directory): assert not download.settings.get('tag_names') -def test_api_download_video_tags(test_session, test_client, tag_factory): +@pytest.mark.asyncio +async def test_api_download_video_tags(test_session, test_async_client, tag_factory): """A user can request Tags for a video being downloaded.""" - tag1 = tag_factory() - tag2 = tag_factory() + tag1 = await tag_factory() + tag2 = await tag_factory() content = dict(urls=['https://example.com/video1', ], downloader='video', settings={'tag_names': [tag1.name, tag2.name]}) - request, response = test_client.post('/api/download', content=json.dumps(content)) + request, response = await test_async_client.post('/api/download', content=json.dumps(content)) assert response.status_code == HTTPStatus.NO_CONTENT download = test_session.query(Download).one() @@ -176,7 +178,9 @@ def test_api_download_video_tags(test_session, test_client, tag_factory): assert download.settings['tag_names'] == [tag1.name, tag2.name] -def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory, assert_video_search): +@pytest.mark.asyncio +async def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory, + assert_video_search): """Test that videos can be searched and that their order is by their textsearch rank.""" # These captions have repeated letters, so they will be higher in the ranking. videos = [ @@ -193,37 +197,38 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit # Repeated runs should return the same result for _ in range(2): # Only videos with a b are returned, ordered by the amount of b's - assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4]) + await assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4]) # Only two captions have e - assert_video_search(search_str='e', assert_ids=[4, 1]) + await assert_video_search(search_str='e', assert_ids=[4, 1]) # Only 2 contains 2 - assert_video_search(search_str='2', assert_ids=[2, ]) + await assert_video_search(search_str='2', assert_ids=[2, ]) # Only two captions have d - assert_video_search(search_str='d', assert_ids=[1, 2]) + await assert_video_search(search_str='d', assert_ids=[1, 2]) # 5 can be gotten by it's title - assert_video_search(search_str='5', assert_ids=[5, ]) + await assert_video_search(search_str='5', assert_ids=[5, ]) # only video 1 has e and d - assert_video_search(search_str='e d', assert_ids=[1, ]) + await assert_video_search(search_str='e d', assert_ids=[1, ]) # video 1 and 4 have b and e, but 1 has more - assert_video_search(search_str='b e', assert_ids=[1, 4]) + await assert_video_search(search_str='b e', assert_ids=[1, 4]) # Check totals are correct even with a limit - assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4) + await assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4) -def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory): +@pytest.mark.asyncio +async def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory): """Search the Video table. This does not need to use a join with the File table.""" vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1') vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2', channel_id=simple_channel.id) vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3') - tag1, tag2 = tag_factory(), tag_factory() + tag1, tag2 = await tag_factory(), await tag_factory() vid1.add_tag(tag1.id) @@ -237,22 +242,68 @@ def test_search_videos(test_session, video_factory, assert_video_search, simple_ assert test_session.query(Video).count() == 3 assert test_session.query(FileGroup).count() == 3 - assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime') - assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime') - assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2) - assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2, - offset=1) - assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime', - channel_id=simple_channel.id) - assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name]) - assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name]) - assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1) + await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime') + await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime') + await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2) + await assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2, + offset=1) + await assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime', + channel_id=simple_channel.id) + await assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name]) + await assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name]) + await assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1) # Only vid2 has both tags. - assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name]) + await assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name]) # No results, no total is returned from the DB. - assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2) + await assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2) # Check all order_by. for order_by in video_lib.VIDEO_ORDERS: - assert_video_search(order_by=order_by) + await assert_video_search(order_by=order_by) + + +@pytest.mark.asyncio +async def test_delete_video(test_async_client, test_session, video_factory, test_download_manager): + """When a Video record is deleted, the FileGroup should be deleted. The URL is added to the + global skip list so the video is not downloaded again.""" + video = video_factory(with_video_file=True, with_poster_ext='png') + video.file_group.url = 'https://example.com/video/1' + files = video.file_group.my_paths() + test_session.commit() + + request, response = await test_async_client.delete(f'/api/videos/video/{video.id}') + assert response.status_code == HTTPStatus.NO_CONTENT + + assert test_session.query(Video).count() == 0, 'Video was not deleted.' + assert test_session.query(FileGroup).count() == 0, 'Video FileGroup was not deleted.' + assert not any(i.exists() for i in files), 'All files should have been deleted.' + assert test_download_manager.is_skipped('https://example.com/video/1'), 'Video should not be downloaded again.' + + +@pytest.mark.asyncio +async def test_format_videos_description(test_async_client, test_session, test_directory, channel_factory, tag_factory): + channel = channel_factory(name='Channel Name') + tag = await tag_factory() + test_session.commit() + + body = dict() + request, response = await test_async_client.post('/api/videos/tag_info', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['videos_destination'] == 'videos' + + body = dict(channel_id=channel.id) + request, response = await test_async_client.post('/api/videos/tag_info', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['videos_destination'] == 'videos/Channel Name' + + body = dict(channel_id=channel.id, tag_name=tag.name) + request, response = await test_async_client.post('/api/videos/tag_info', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['videos_destination'] == 'videos/one/Channel Name' + + get_wrolpi_config().videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s' + body = dict(channel_id=channel.id, tag_name=tag.name) + request, response = await test_async_client.post('/api/videos/tag_info', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name' diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 00044b5e..a0ce7ed7 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -135,10 +135,12 @@ def test_generate_video_poster(video_file): @pytest.mark.asyncio -async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config): +async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config, + tag_factory): """Importing the Channels' config should create any missing download records""" channel1 = channel_factory(source_id='foo', url='https://example.com/channel1') channel2 = channel_factory(source_id='bar', url='https://example.com/channel2') + tag = await tag_factory() assert len(test_session.query(Channel).all()) == 2 test_session.commit() @@ -228,8 +230,19 @@ async def test_import_channel_downloads(test_async_client, test_session, channel assert {i.url for i in channel1.downloads} == {'https://example.com/channel1'} assert {i.url for i in channel2.downloads} == {'https://example.com/channel2', 'https://example.org'} + # A Channel with a `tag_name` is tagged. + update_channel_config(get_channels_config(), + channel1.source_id, + {'tag_name': tag.name}) + import_channels_config() + channel1, channel2 = test_session.query(Channel).order_by(Channel.url).all() + assert channel1.tag == tag and channel1.tag_id == tag.id + assert channel2.tag is None and channel2.tag_id is None -def test_import_channel_delete_missing_channels(test_session, channel_factory, test_channels_config): + +@pytest.mark.asyncio +async def test_import_channel_delete_missing_channels( + test_async_client, test_session, channel_factory, test_channels_config): """The Channel import function deletes any Channels that are not in the config.""" # Create a DB and config with two Channels. channel1 = channel_factory(source_id='foo') @@ -268,32 +281,6 @@ def test_import_channel_delete_missing_channels(test_session, channel_factory, t assert str(channel2.directory) not in test_channels_config.read_text() -def test_check_for_video_corruption(video_file, test_directory): - # The test video is not corrupt. - assert common.check_for_video_corruption(video_file) is False - - # An empty file is corrupt. - empty_file = test_directory / 'empty file.mp4' - empty_file.touch() - assert common.check_for_video_corruption(empty_file) is True - - # A video file must be complete. - truncated_video = test_directory / 'truncated_video.mp4' - with truncated_video.open('wb') as fh: - fh.write(video_file.read_bytes()[:10000]) - assert common.check_for_video_corruption(truncated_video) is True - - # Check for specific ffprobe errors. - with mock.patch('modules.videos.common.subprocess') as mock_subprocess: - # `video_file` is ignored for these calls. - mock_subprocess.run().stderr = b'Something\nInvalid NAL unit size' - assert common.check_for_video_corruption(video_file) is True - mock_subprocess.run().stderr = b'Something\nError splitting the input into NAL units' - assert common.check_for_video_corruption(video_file) is True - mock_subprocess.run().stderr = b'Some stderr is fine' - assert common.check_for_video_corruption(video_file) is False - - @pytest.mark.asyncio async def test_ffprobe_json(test_async_client, video_file, corrupted_video_file): content = await common.ffprobe_json(video_file) diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index d432591d..13597176 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -107,7 +107,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f mock_video_extract_info, mock_video_process_runner, tag_factory): """A Video is tagged when Download record requires it.""" video_path = video_file_factory() - tag1, tag2 = tag_factory(), tag_factory() + tag1, tag2 = await tag_factory(), await tag_factory() url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' mock_video_extract_info.return_value = copy(example_video_json) diff --git a/modules/videos/test/test_models.py b/modules/videos/test/test_models.py index f1681765..19a2a487 100644 --- a/modules/videos/test/test_models.py +++ b/modules/videos/test/test_models.py @@ -34,7 +34,7 @@ async def test_channel_move(test_async_client, test_session, test_directory, cha foo.mkdir(parents=True) # Move the Channel. - await channel.move(foo, test_session) + await channel.move_channel(foo, test_session) assert channel.directory == foo assert channel.directory != (test_directory / 'New Channel Directory') diff --git a/modules/videos/test/test_video.py b/modules/videos/test/test_video.py index 5d3c5537..a081ea6b 100644 --- a/modules/videos/test/test_video.py +++ b/modules/videos/test/test_video.py @@ -29,27 +29,11 @@ def test_delete_video_no_channel(test_session, simple_video): assert test_session.query(Video).count() == 0 -def test_delete_video(test_session, video_factory): - """When a Video record is deleted, all referenced file records should be deleted.""" - video = video_factory(with_video_file=True, with_poster_ext='png') - test_session.commit() - - assert video.video_path.is_file(), 'Video file was not created.' - assert video.poster_path.is_file(), 'Video poster was not created.' - assert test_session.query(Video).count() == 1, 'Video was not created.' - assert test_session.query(FileGroup).count() == 1, 'Video file and poster file were not created.' - - video.delete() - test_session.commit() - assert test_session.query(Video).count() == 0, 'Video was not deleted.' - assert test_session.query(FileGroup).count() == 0, 'Video files were not deleted.' - - @pytest.mark.asyncio async def test_delete_video_with_tag(test_async_client, test_session, video_factory, tag_factory): """You cannot delete a video with a tag.""" video = video_factory(with_video_file=True, with_poster_ext='png') - tag = tag_factory() + tag = await tag_factory() video.add_tag(tag.name) test_session.commit() @@ -63,7 +47,7 @@ async def test_delete_video_with_tag(test_async_client, test_session, video_fact @pytest.mark.asyncio -async def test_video_channel_refresh(test_session, test_directory, channel_factory, video_factory): +async def test_video_channel_refresh(test_async_client, test_session, test_directory, channel_factory, video_factory): """A Video will be associated with a Channel when it's files are in that Channel's directory.""" # Create a video file in this channel's directory. channel = channel_factory() @@ -101,7 +85,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]} entry = channel.info_json['entries'][0] - tag = tag_factory() + tag = await tag_factory() vid1 = video_factory( channel_id=channel.id, title=f'{channel.name}_20000101_ABC123456_The video title', @@ -150,7 +134,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto @pytest.mark.asyncio -async def test_delete_renamed_video(test_session, channel_factory, video_factory, tag_factory): +async def test_delete_renamed_video(test_async_client, test_session, channel_factory, video_factory, tag_factory): """If duplicate Video's exist, delete all the videos that do not have the new title.""" channel = channel_factory(name='Channel Name') video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4' @@ -158,7 +142,7 @@ async def test_delete_renamed_video(test_session, channel_factory, video_factory channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'Some new title'}]} entry = channel.info_json['entries'][0] - tag = tag_factory() + tag = await tag_factory() vid1 = video_factory( channel_id=channel.id, title=f'{channel.name}_20000101_ABC123456_The video title', diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index 79775c1e..0a7f8fb0 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -164,8 +164,11 @@ def delete_videos(*video_ids: int, session: Session = None): raise UnknownVideo('Could not find videos to delete') logger.warning(f'Deleting {len(videos)} videos') + # Get all URLs, skip them once (so config isn't saved multiple times). + urls = list(filter(None, [i.file_group.url for i in videos])) + download_manager.add_to_skip_list(*urls) for video in videos: - video.delete(add_to_skip_list=True) + video.delete() session.commit() diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py index 7058657f..f2806f26 100644 --- a/modules/videos/video/test/test_api.py +++ b/modules/videos/video/test/test_api.py @@ -9,7 +9,7 @@ from modules.videos.models import Video from wrolpi.dates import now -def test_get_video_prev_next(test_session, channel_factory, video_factory): +def test_get_video_prev_next(test_async_client, test_session, channel_factory, video_factory): """ Test that the previous and next videos will be retrieved when fetching a video. """ @@ -69,7 +69,7 @@ def test_get_video_prev_next(test_session, channel_factory, video_factory): raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e -def test_get_video_prev_next_no_upload_date(test_session, video_factory, channel_factory): +def test_get_video_prev_next_no_upload_date(test_async_client, test_session, video_factory, channel_factory): """ Previous and Next Videos should be those next to a Video file when videos do not have upload dates. """ @@ -153,7 +153,7 @@ async def test_wrol_mode(test_async_client, simple_channel, simple_video, wrol_m test_download_manager, tag_factory): """Many methods are blocked when WROL Mode is enabled.""" channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory))) - tag = tag_factory() + tag = await tag_factory() await wrol_mode_fixture(True) diff --git a/modules/zim/conftest.py b/modules/zim/conftest.py index 5309cf78..16096d5c 100644 --- a/modules/zim/conftest.py +++ b/modules/zim/conftest.py @@ -100,14 +100,14 @@ def zim_factory(test_session, zim_path_factory, test_directory): @pytest.fixture -def assert_zim_search(test_client, test_directory): +def assert_zim_search(test_async_client, test_directory): from http import HTTPStatus - def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [], - expected_status_code: int = HTTPStatus.OK): + async def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [], + expected_status_code: int = HTTPStatus.OK): content = dict(search_str=search_str, tag_names=tag_names) assert isinstance(zim_id, int), 'This fixture only supports Zim ids of type integer' - request, response = test_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content)) + request, response = await test_async_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content)) assert response.status_code == expected_status_code result = response.json['zim'] diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 4e9a3936..7c1d6492 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -16,7 +16,7 @@ from modules.zim.kiwix import KIWIX_CATALOG from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription from wrolpi import flags from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \ - register_refresh_cleanup, background_task, get_wrolpi_config + register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate from wrolpi.db import get_db_session, optional_session, get_db_curs from wrolpi.downloader import DownloadFrequency from wrolpi.files.lib import refresh_files, split_file_name_words @@ -124,14 +124,11 @@ def get_entries_tags(paths: List[str], zim_id: int): def get_unique_paths(*paths: str) -> List[str]: """Return a new list which contains unique Zim paths.""" - new_paths = list() - simple_paths = list() - for path in paths: - simple_path = path.replace('-', '').replace('_', '').replace('/', '').lower() - if simple_path not in simple_paths: - new_paths.append(path) - simple_paths.append(simple_path) - return new_paths + unique_paths = unique_by_predicate( + paths, + lambda i: i.replace('-', '').replace('_', '').replace('/', '').lower() + ) + return list(unique_paths) @optional_session diff --git a/modules/zim/test/test_api.py b/modules/zim/test/test_api.py index 9bf95580..49151037 100644 --- a/modules/zim/test/test_api.py +++ b/modules/zim/test/test_api.py @@ -8,16 +8,17 @@ from modules.zim.models import Zim from wrolpi.downloader import Download -def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim_search): +@pytest.mark.asyncio +async def test_zim_search(test_async_client, test_session, test_zim, tag_factory, assert_zim_search): """Entries can be tagged and retrieved using their containing Zim.""" - tag1, tag2 = tag_factory(), tag_factory() + tag1, tag2 = await tag_factory(), await tag_factory() test_zim.tag_entry(tag1.name, 'one') test_zim.tag_entry(tag2.name, 'one') test_zim.tag_entry(tag1.name, 'two') test_session.commit() # Test searching with only `search_str`. - assert_zim_search('one', 1, { + await assert_zim_search('one', 1, { 'path': test_zim.path, 'search': [ # Entry "One" contains a 'one' in the title, and the text. @@ -27,13 +28,13 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim ], 'estimate': 2, }) - assert_zim_search('two', 1, { + await assert_zim_search('two', 1, { 'path': test_zim.path, 'search': [{'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271}, {'path': 'home', 'headline': 'testing.\nOne\nTwo', 'rank': 0.06079271}], 'estimate': 2, }) - assert_zim_search('item', 1, { + await assert_zim_search('item', 1, { 'path': test_zim.path, 'search': [{'path': 'one', 'headline': 'first item', 'rank': 0.06079271}, {'path': 'two', 'headline': 'second item', 'rank': 0.06079271}], @@ -41,35 +42,35 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim }) # Test searching with `search_str` and `tag_names`. - assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={ + await assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={ 'search': [ # Only entry "One" is tagged with Tag 2. {'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271}, ], 'estimate': 1, }) - assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={ + await assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={ 'search': [ # Only "One" is tagged with Tag 1 and contains "one". {'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271}, ], 'estimate': 1, }) - assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={ + await assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={ 'search': [ # Only "Two" was tagged with Tag 1 and contains "two". {'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271}, ], 'estimate': 1, }) - assert_zim_search('two', 1, tag_names=[tag2.name, ], expected={ + await assert_zim_search('two', 1, tag_names=[tag2.name, ], expected={ # Two was not tagged with Tag 1. 'search': [], 'estimate': 0, }) # Test search with only `tag_names`. - assert_zim_search('', 1, tag_names=[tag1.name, tag2.name], expected={ + await assert_zim_search('', 1, tag_names=[tag1.name, tag2.name], expected={ 'search': [ # Return all entries that are tagged with the Tags. {'path': 'one', 'headline': 'This is the first item', 'rank': 0.0, 'tag_names': ['one', 'two']}, @@ -78,9 +79,10 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim }) -def test_entries_by_tag(test_client, test_session, test_zim, tag_factory): +@pytest.mark.asyncio +async def test_entries_by_tag(test_async_client, test_session, test_zim, tag_factory): """Entries can be tagged and retrieved using their containing Zim.""" - tag1, tag2 = tag_factory(), tag_factory() + tag1, tag2 = await tag_factory(), await tag_factory() test_zim.tag_entry(tag1.name, 'one') test_zim.tag_entry(tag2.name, 'one') test_zim.tag_entry(tag1.name, 'two') @@ -96,36 +98,38 @@ def test_entries_by_tag(test_client, test_session, test_zim, tag_factory): assert [(i.path, i.title) for i in result] == [('one', 'One'), ] -def test_zim_crud(test_client, test_session, test_directory, test_zim): - request, response = test_client.get('/api/zim') +@pytest.mark.asyncio +async def test_zim_crud(test_async_client, test_session, test_directory, test_zim): + request, response = await test_async_client.get('/api/zim') assert response.status_code == HTTPStatus.OK assert len(response.json['zims']) == 1 assert response.json['zims'][0]['path'] == test_zim.path.name - request, response = test_client.delete('/api/zim/1') + request, response = await test_async_client.delete('/api/zim/1') assert response.status_code == HTTPStatus.NO_CONTENT assert not test_zim.path.exists() assert test_session.query(Zim).count() == 0 -def test_zim_tag_and_untag(test_client, test_session, test_zim, tag_factory): +@pytest.mark.asyncio +async def test_zim_tag_and_untag(test_async_client, test_session, test_zim, tag_factory): """Zim entries can be Tagged and Untagged.""" - tag1, tag2 = tag_factory(), tag_factory() + tag1, tag2 = await tag_factory(), await tag_factory() test_session.commit() # Entries can be tagged in the API. content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home') - request, response = test_client.post('/api/zim/tag', content=json.dumps(content)) + request, response = await test_async_client.post('/api/zim/tag', content=json.dumps(content)) assert response.status_code == HTTPStatus.CREATED # Entries can be untagged in the API. content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home') - request, response = test_client.post('/api/zim/untag', content=json.dumps(content)) + request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content)) assert response.status_code == HTTPStatus.NO_CONTENT # Cannot untag twice. content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home') - request, response = test_client.post('/api/zim/untag', content=json.dumps(content)) + request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content)) assert response.status_code == HTTPStatus.NOT_FOUND diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index 7899f9fd..42fef7f0 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -51,14 +51,14 @@ async def test_zim_get_entry(test_async_client, test_session, zim_path_factory): @pytest.mark.asyncio async def test_zim_bad_entry(test_async_client, test_session, test_zim, tag_factory): - tag1 = tag_factory() + tag1 = await tag_factory() with pytest.raises(UnknownZimEntry): test_zim.tag_entry(tag1.name, 'path does not exist') @pytest.mark.asyncio async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, tag_factory): - tag1, tag2 = tag_factory('tag1'), tag_factory('tag2') + tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2') test_zim.tag_entry(tag1.name, 'one') test_zim.tag_entry(tag2.name, 'one') test_zim.tag_entry(tag1.name, 'two') @@ -70,7 +70,8 @@ async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, t @pytest.mark.asyncio async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory, tag_factory): - tze = zim_factory('zim1').tag_entry(tag_factory().name, 'one') + tag = await tag_factory() + tze = zim_factory('zim1').tag_entry(tag.name, 'one') # Test TagZimEntry.__repr__ assert repr(tze) == "" @@ -78,7 +79,7 @@ async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory, @pytest.mark.asyncio async def test_zim_entries_with_tags(test_async_client, test_session, zim_factory, tag_factory): """Can fetch Zim Entries that have Tags. Can filter only those entries that are tagged on a specific Zim.""" - tag1, tag2 = tag_factory(), tag_factory() + tag1, tag2 = await tag_factory(), await tag_factory() zim1, zim2 = zim_factory('zim1'), zim_factory('zim2') # Tag Zim1 with both tags, but Zim2 shares one of the tags and is returned when no Zim is passed. zim1.tag_entry(tag1.name, 'one') @@ -124,13 +125,14 @@ def test_get_unique_paths(): 'A/Thomas_Harvey_Johnston'] -def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now): +@pytest.mark.asyncio +async def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now): fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1)) config = tags.get_tags_config() assert not config.tag_zims # Tag three Zim entries. - tag1, tag2 = tag_factory('Tag1'), tag_factory('Tag2') + tag1, tag2 = await tag_factory('Tag1'), await tag_factory('Tag2') test_zim.tag_entry(tag1.name, 'one') test_zim.tag_entry(tag2.name, 'one') test_zim.tag_entry(tag1.name, 'two') @@ -243,12 +245,13 @@ async def test_zim_download(test_session, kiwix_download_zim, test_directory, te assert not flags.outdated_zims.is_set() -def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config): +@pytest.mark.asyncio +async def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config): """When an outdated Zim file is deleted, the Tags should be moved to the latest Zim.""" zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim. zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim') zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim. - tag1, tag2 = tag_factory('tag1'), tag_factory('tag2') + tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2') # Tag outdated Zim entries. zim1.tag_entry(tag1.name, 'home') zim1.tag_entry(tag2.name, 'one') diff --git a/wrolpi/common.py b/wrolpi/common.py index e9fb2e9e..d5b1948b 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -14,6 +14,7 @@ import socket import string import sys import tempfile +import traceback from asyncio import Task from copy import deepcopy from dataclasses import dataclass @@ -160,6 +161,7 @@ __all__ = [ 'ConfigFile', 'DownloadFileInfo', 'DownloadFileInfoLink', + 'LOGGING_CONFIG', 'ModelHelper', 'WROLPI_CONFIG', 'aiohttp_get', @@ -180,7 +182,6 @@ __all__ = [ 'compile_tsvector', 'cum_timer', 'date_range', - 'LOGGING_CONFIG', 'disable_wrol_mode', 'download_file', 'enable_wrol_mode', @@ -227,6 +228,7 @@ __all__ = [ 'truncate_generator_bytes', 'truncate_object_bytes', 'tsvector', + 'unique_by_predicate', 'url_strip_host', 'walk', 'wrol_mode_check', @@ -252,10 +254,6 @@ class ModelHelper: def get_by_path(path, session) -> Base: raise NotImplementedError('This model has not defined this method.') - @staticmethod - def find_by_paths(paths, session) -> List: - raise NotImplementedError('This model has not defined this method.') - @staticmethod def get_by_id(id_: int, session: Session = None) -> Optional[Base]: """Attempts to get a model instance by its id. Returns None if no instance can be found.""" @@ -274,9 +272,10 @@ class ModelHelper: def do_model(file_group, session: Session): raise NotImplementedError('This model has not defined this method.') - def flush(self): + def flush(self, session: Session = None): """A convenience method which flushes this record using its DB Session.""" - if session := Session.object_session(self): + session = session or Session.object_session(self) + if session: session.flush([self]) else: raise RuntimeError(f'{self} is not in a session!') @@ -746,6 +745,9 @@ def run_after(after: callable, *args, **kwargs) -> callable: synchronous_after = after async def after(*a, **kw): + if logger.isEnabledFor(logging.DEBUG): + traceback.print_stack() + logger.debug(f'run_after synchronous_after called for {after}') return synchronous_after(*a, **kw) def wrapper(func: callable): @@ -756,6 +758,9 @@ def run_after(after: callable, *args, **kwargs) -> callable: @wraps(func) async def wrapped(*a, **kw): results = await func(*a, **kw) + if logger.isEnabledFor(logging.DEBUG): + traceback.print_stack() + logger.debug(f'run_after async called for {func}') coro = after(*args, **kwargs) asyncio.ensure_future(coro) return results @@ -763,6 +768,9 @@ def run_after(after: callable, *args, **kwargs) -> callable: @wraps(func) def wrapped(*a, **kw): results = func(*a, **kw) + if logger.isEnabledFor(logging.DEBUG): + traceback.print_stack() + logger.debug(f'run_after sync called for {func}') coro = after(*args, **kwargs) asyncio.ensure_future(coro) return results @@ -1741,6 +1749,35 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl return headlines +async def search_other_estimates(tag_names: List[str]) -> dict: + """Estimate other things that are Tagged.""" + from wrolpi.db import get_db_curs + + if not tag_names: + return dict( + channel_count=0, + ) + + with get_db_curs() as curs: + stmt = ''' + SELECT + COUNT(c.id) + FROM channel c + LEFT OUTER JOIN public.tag t on t.id = c.tag_id + WHERE t.name = %(tag_name)s + ''' + # TODO handle multiple tags + params = dict(tag_name=tag_names[0]) + curs.execute(stmt, params) + + channel_count = curs.fetchone()[0] + + others = dict( + channel_count=channel_count, + ) + return others + + def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False): """Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful. The original file will be restored if any of these steps fail.""" @@ -1891,3 +1928,29 @@ def is_valid_hex_color(hex_color: str) -> bool: def is_hardlinked(path: pathlib.Path) -> bool: """Returns True only if `path` has more than one link.""" return path.stat().st_nlink > 1 + + +def _default_predicate(i): + return i + + +def unique_by_predicate( + iterable: list | tuple | set, + predicate: callable = None, +) -> list | tuple | set: + """Finds only unique items in the provided iterable based on the predicate function. Order is preserved. + + `predicate` defaults to the items themselves. + + >>> unique_by_predicate([['apple', 'banana', 'pear', 'kiwi', 'plum', 'peach']], lambda i: len(i)) + # [ 'apple', 'banana', 'pear' ] + >>> unique_by_predicate([Path('foo.txt'), Path('foo.mp4'), Path('bar.txt')], lambda i: i.stem) + # [ Path('foo.txt'), Path('bar.txt') ] + >>> unique_by_predicate([1, 1, 2, 3, 3, 3, 3, 4], None) + # [ 1, 2, 3, 4 ] + """ + predicate = predicate or _default_predicate # Return an iterable based off the object itself by default. + seen = set() + unique_items = [i for i in iterable if not ((key := predicate(i)) in seen or seen.add(key))] + # Return the same type as the input iterable + return iterable.__class__(unique_items) # For other iterable types diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 2492e107..8d1c4257 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -16,7 +16,7 @@ import zipfile from abc import ABC from datetime import datetime from itertools import zip_longest -from typing import List, Callable, Dict, Sequence, Union, Coroutine +from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional from typing import Tuple, Set from unittest import mock from unittest.mock import MagicMock, AsyncMock @@ -175,7 +175,7 @@ async def test_async_client(test_directory) -> SanicASGITestClient: @pytest.fixture -def test_download_manager_config(test_async_client, test_directory): +def test_download_manager_config(test_async_client, test_directory) -> pathlib.Path: with downloads_manager_config_context(): (test_directory / 'config').mkdir(exist_ok=True) config_path = test_directory / 'config/download_manager.yaml' @@ -187,7 +187,7 @@ async def test_download_manager( test_async_client, test_session, # session is required because downloads can start without the test DB in place. test_download_manager_config, -): +) -> DownloadManager: # Needed to use signals in test app? api_app.signalize() @@ -200,7 +200,7 @@ async def test_download_manager( @pytest.fixture -def fake_now(): +def fake_now() -> Callable: try: set_test_now(datetime(2000, 1, 1)) yield set_test_now @@ -210,17 +210,17 @@ def fake_now(): @pytest.fixture -def successful_download(): +def successful_download() -> DownloadResult: return DownloadResult(success=True) @pytest.fixture -def failed_download(): - return DownloadResult(error='pytest.fixture failed_download error', success=False) +def failed_download() -> DownloadResult: + return DownloadResult(error='wrolpi.conftest failed_download error', success=False) @pytest.fixture -def assert_download_urls(test_session): +def assert_download_urls(test_session) -> Callable[[Set[str]], None]: def asserter(expected_urls: Set[str]): downloads = test_session.query(Download).all() urls = {i.url for i in downloads} @@ -231,7 +231,7 @@ def assert_download_urls(test_session): @pytest.fixture -def assert_downloads(test_session): +def assert_downloads(test_session) -> Callable[[List[Dict]], None]: from wrolpi.test.common import assert_dict_contains def asserter(expected: List[Dict]): @@ -245,7 +245,7 @@ def assert_downloads(test_session): @pytest.fixture -def test_downloader(test_download_manager): +def test_downloader(test_download_manager) -> Downloader: class TestDownloader(Downloader, ABC): """A testing Downloader""" name = 'test_downloader' @@ -314,7 +314,7 @@ def corrupted_video_file(test_directory) -> pathlib.Path: @pytest.fixture -def video_file_factory(test_directory): +def video_file_factory(test_directory) -> Callable[[pathlib.Path], pathlib.Path]: """Return a copy of the example Big Buck Bunny video in the `test_directory`.""" def _(path: pathlib.Path = None) -> pathlib.Path: @@ -327,7 +327,7 @@ def video_file_factory(test_directory): @pytest.fixture -def video_bytes(): +def video_bytes() -> bytes: return (PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes() @@ -467,7 +467,7 @@ def wrol_mode_fixture(test_config, test_download_manager) -> Callable[[bool], Co @pytest.fixture -def mock_create_subprocess_shell(): +def mock_create_subprocess_shell() -> Callable: def mocker(communicate_return=None, return_code=None, communicate_side_effect=None): async def communicate(*a, **kw): if communicate_side_effect: @@ -507,14 +507,14 @@ def flags_lock(): @pytest.fixture() -def tag_factory(test_session): +def tag_factory(test_session) -> Callable[[Optional[str]], Awaitable[Tag]]: names = ['one', 'two', 'three', 'four', 'five', 'six'] count = 1 - def factory(name: str = None) -> Tag: + async def factory(name: str = None) -> Tag: if not name: name = names.pop(0) - tag = upsert_tag(name, f'#{str(count) * 6}') + tag = await upsert_tag(name, f'#{str(count) * 6}', session=test_session) return tag return factory diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index aa460aa8..08959b5e 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -52,6 +52,7 @@ def attach_shared_contexts(app: Sanic): # Locks app.shared_ctx.config_save_lock = multiprocessing.Lock() + app.shared_ctx.events_lock = multiprocessing.Lock() reset_shared_contexts(app) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index c0636a23..0fbefa56 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -253,6 +253,15 @@ class Download(ModelHelper, Base): # noqa return download raise UnknownDownload(f'Cannot find Download with URL: {url}') + @classmethod + @optional_session + def get_all_by_destination(cls, destination: str | pathlib.Path, session: Session = None) -> List['Download']: + destination = str(destination) + downloads = session.query(Download) \ + .filter(Download.settings.contains({'destination': destination})) \ + .all() + return downloads + class Downloader: name: str = None @@ -1330,7 +1339,7 @@ class RSSDownloader(Downloader, ABC): # Only download new URLs. urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(*urls)]] - logger.info(f'Successfully got {len(urls)} urls from rss {download.url}') + logger.info(f'Successfully got {len(urls)} new URLs from RSS {download.url}') return DownloadResult(success=True, downloads=urls) diff --git a/wrolpi/events.py b/wrolpi/events.py index 694d0302..7a55ddd8 100644 --- a/wrolpi/events.py +++ b/wrolpi/events.py @@ -10,7 +10,6 @@ from wrolpi.dates import now logger = logger.getChild(__name__) HISTORY_SIZE = 100 -EVENTS_LOCK = multiprocessing.Lock() class Events: @@ -79,6 +78,14 @@ class Events: def send_shutdown_failed(message: str = None): send_event('shutdown_failed', message, subject='shutdown') + @staticmethod + def send_file_move_completed(message: str = None): + send_event('file_move_completed', message, subject='refresh') + + @staticmethod + def send_file_move_failed(message: str = None): + send_event('file_move_failed', message, subject='refresh') + def log_event(event: str, message: str = None, action: str = None, subject: str = None): log = f'{event=}' @@ -93,7 +100,7 @@ def log_event(event: str, message: str = None, action: str = None, subject: str def send_event(event: str, message: str = None, action: str = None, subject: str = None, url: str = None): from wrolpi.api_utils import api_app - EVENTS_LOCK.acquire() + api_app.shared_ctx.events_lock.acquire() try: # All events will be in time order, they should never be at the exact same time. dt = now() @@ -112,7 +119,7 @@ def send_event(event: str, message: str = None, action: str = None, subject: str while len(api_app.shared_ctx.events_history) > HISTORY_SIZE: api_app.shared_ctx.events_history.pop(0) finally: - EVENTS_LOCK.release() + api_app.shared_ctx.events_lock.release() log_event(event, message, action, subject) diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 046bba9d..6b01c0cb 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -12,6 +12,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_ from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict from . import lib, schema from ..api_utils import json_response, api_app +from ..events import Events from ..schema import JSONErrorResponse from ..tags import Tag from ..vars import PYTEST @@ -224,7 +225,9 @@ async def post_move(_: Request, body: schema.Move): try: await lib.move(destination, *sources) + Events.send_file_move_completed(f'Moving files to {body.destination} succeeded') except Exception as e: + Events.send_file_move_failed(f'Moving files to {body.destination} failed!') raise FileConflict(f'Failed to move {sources} to {destination}') from e return response.empty(HTTPStatus.NO_CONTENT) @@ -245,7 +248,7 @@ async def post_rename(_: Request, body: schema.Rename): raise FileConflict(f'File already exists: {new_path}') try: - await lib.rename(path, body.new_name) + await lib.rename(path, body.new_name, send_events=True) except Exception as e: logger.error(f'Failed to rename {path} to {new_path}', exc_info=e) raise FileConflict(f'Failed to rename {path} to {new_path}') from e @@ -297,14 +300,15 @@ async def post_upload(request: Request): Will not overwrite an existing file, unless a previous upload did not complete.""" try: - destination = request.form['destination'][0] + destination_str = request.form['destination'][0] except Exception as e: logger.error(f'Failed to get upload destination', exc_info=e) raise UnknownDirectory('Must provide destination') from e - destination = get_media_directory() / destination - if not destination.is_dir(): - raise UnknownDirectory(f'Destination must be a relative directory that is already in the media directory: {destination}') + destination = get_media_directory() / destination_str + if destination_str.startswith('/') or destination_str.startswith('.') or not destination.is_dir(): + msg = f'Destination must be a relative directory that is already in the media directory: {destination}' + raise UnknownDirectory(msg) try: filename = str(request.form['filename'][0]) @@ -385,7 +389,11 @@ async def post_upload(request: Request): logger.info(f'Got final chunk of uploaded file: {output_str}') del api_app.shared_ctx.uploaded_files[output_str] # Upsert this new file (and any related files) into the DB. - background_task(lib.upsert_file(output, tag_names=tag_names)) + coro = lib.upsert_file(output, tag_names=tag_names) + if PYTEST: + await coro + else: + background_task(coro) return response.empty(HTTPStatus.CREATED) # Request the next chunk. diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index e6fb16a0..376cea53 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -9,13 +9,15 @@ import pathlib import re import shutil import subprocess +from collections import OrderedDict from itertools import zip_longest from pathlib import Path -from typing import List, Tuple, Union, Dict, Generator, Iterable +from typing import List, Tuple, Union, Dict, Generator, Iterable, Set import cachetools.func import psycopg2 from sqlalchemy import asc +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from wrolpi import flags @@ -24,7 +26,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co partition, cancelable_wrapper, \ get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \ get_wrolpi_config, \ - timer + timer, chunks, unique_by_predicate from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \ @@ -32,7 +34,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag from wrolpi.events import Events from wrolpi.files.models import FileGroup, Directory from wrolpi.lang import ISO_639_CODES -from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where +from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config from wrolpi.vars import PYTEST try: @@ -48,7 +50,7 @@ logger = logger.getChild(__name__) __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files', 'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics', - 'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', + 'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem', 'move', 'rename', 'delete_directory', 'handle_file_group_search_results'] @@ -311,6 +313,12 @@ def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = Fals return path.name, '' +def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...] | Set[pathlib.Path]) \ + -> List[pathlib.Path]: + """Returns the first of each Path with a unique stem. Used to detect if a group of files share a stem.""" + return unique_by_predicate(files, lambda i: split_path_stem_and_suffix(i)[0]) + + refresh_logger = logger.getChild('refresh') @@ -578,6 +586,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr refreshing_all_files = False with flags.refreshing, timer('refresh_files', 'info'): + api_app.shared_ctx.refresh['counted_files'] = 0 if not paths: refresh_logger.warning('Refreshing all files') refreshing_all_files = True @@ -603,8 +612,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr continue directories.extend(dirs) found_directories |= set(dirs) - api_app.shared_ctx.refresh['counted_files'] = api_app.shared_ctx.refresh.get('counted_files', - 0) + len(files) + add_files_to_refresh_count(len(files)) # Sleep to catch cancel. await asyncio.sleep(0) refresh_logger.info(f'Counted {api_app.shared_ctx.refresh["counted_files"]} files') @@ -633,7 +641,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr with get_db_session() as session: parent_directories = {i[0] for i in session.query(Directory.path).filter(Directory.path.in_(paths))} parent_directories |= set(filter(lambda i: i.is_dir(), paths)) - await upsert_directories(parent_directories, found_directories) + upsert_directories(parent_directories, found_directories) if send_events: Events.send_global_after_refresh_completed() @@ -684,7 +692,7 @@ async def apply_indexers(): break -async def upsert_directories(parent_directories, directories): +def upsert_directories(parent_directories, directories): """ Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in `directories`. @@ -1168,7 +1176,17 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi return 0 -def _move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]: +MOVE_CHUNK_SIZE = 100 + + +def add_files_to_refresh_count(count: int): + from wrolpi.api_utils import api_app + count = api_app.shared_ctx.refresh.get('counted_files', 0) + count + api_app.shared_ctx.refresh['counted_files'] = count + + +async def _move(destination: pathlib.Path, *sources: pathlib.Path, session: Session) \ + -> OrderedDict[pathlib.Path, pathlib.Path]: media_directory = get_media_directory() for source in sources: if not str(source).startswith(str(media_directory)): @@ -1177,70 +1195,155 @@ def _move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathl raise FileNotFoundError(f'{destination} is not within the media directory') destination_existed = destination.is_dir() + destination.mkdir(parents=True, exist_ok=True) - # [ (pathlib.Path('source'), pathlib.Path('destination')), ... ] - plan = list() - unique_paths = set() - # Get all files that need to be moved. Check for destination conflicts. - for source in sources: - if source.is_dir(): - # Recursively move the directory's files. - for path in walk(source): - # Preserve the directory structure that contains this file. - new_file = destination / source.name / path.relative_to(source) - if new_file.exists(): - raise FileConflict(f'Cannot move {path} to {new_file} because it already exists') - if path not in unique_paths: - plan.append((path, new_file)) - unique_paths.add(path) - elif source.is_file(): - # Move all files associated with the source file. - for file in glob_shared_stem(source): - new_file = destination / file.name - if new_file.exists(): - raise FileConflict(f'Cannot move {file} to {new_file} because it already exists') - if file not in unique_paths: - plan.append((file, new_file)) - unique_paths.add(file) + # The files that will be moved. [ (old_file, new_file), ... ] + plan: Dict[pathlib.Path, pathlib.Path] = dict() - logger.info(f'Executing move plan with {len(plan)} steps.') + def add_file_group_to_plan(file_group_: FileGroup): + primary_path_ = file_group_.primary_path + if primary_path_ not in plan: + new_primary_path_ = destination / primary_path_.name + if new_primary_path_.exists(): + raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}') + plan[primary_path_] = new_primary_path_ - # Track the steps performed so they can be reversed. - revert_plan = list() + def add_source_file_group_to_plan(file_group_: FileGroup, source_: pathlib.Path): + primary_path_ = file_group_.primary_path + if primary_path_ not in plan: + new_primary_path_ = destination / source_.name / primary_path_.relative_to(source) + if new_primary_path_.exists(): + raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}') + plan[primary_path_] = new_primary_path_ - def do_plan(plan_): - # Move deepest files first. - plan_ = sorted(plan_, key=lambda i: (len(i[0].parents), i[0].name), reverse=True) - for old_file, new_file in plan_: # noqa - new_file.parent.mkdir(parents=True, exist_ok=True) - logger.debug(f'Moving {old_file} to {new_file}') - # if not new_file.exists(): - shutil.move(str(old_file), str(new_file)) - logger.debug(f'Moved {old_file} to {new_file}') - revert_plan.append((new_file, old_file)) - # Move is complete. - return plan_ + # Sources may be a list of files, we only want to issue a query for one file in each FileGroup. + sources = get_unique_files_by_stem(sources) + logger.info(f'move got {len(sources)} to move') - try: - plan = do_plan(plan) + with flags.refresh_counting: for source in sources: - if source.is_dir(): - delete_directory(source) - logger.info(f'Move execution completed') - return plan - except Exception as e: - logger.error(f'Move failed', exc_info=e) - # Move files back. - do_plan(revert_plan) - # Delete the directories that were created. - directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True) - for path in directories: - if path.is_dir(): - delete_directory(path) - # Remove destination only if it did not exist at the start. - if destination.is_dir() and destination_existed is False: - delete_directory(destination) - raise + if source.is_file(): + # Get any FileGroups that share the source's stem. + files = glob_shared_stem(source) + file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(files)).all() + if not file_groups: + # No FileGroups for this source file, create one. + try: + fg = FileGroup.from_paths(session, *files) + file_groups = [fg, ] + except NoPrimaryFile: + for file in files: + fg = FileGroup.from_paths(session, file) + add_file_group_to_plan(fg) + add_files_to_refresh_count(len(file_groups)) + for fg in file_groups: + add_file_group_to_plan(fg) + # Sleep to catch cancel. + await asyncio.sleep(0) + elif source.is_dir(): + stmt = 'SELECT primary_path FROM file_group WHERE primary_path LIKE :like' + for (primary_path,) in session.execute(stmt, dict(like=f'{source}%')): + primary_path = pathlib.Path(primary_path) + if primary_path not in plan: + new_primary_path = destination / source.name / primary_path.relative_to(source) + if new_primary_path.exists(): + raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path}') + plan[primary_path] = new_primary_path + add_files_to_refresh_count(1) + + # Sleep to catch cancel. + await asyncio.sleep(0) + + # Move directories of this source. + for directory in (i for i in walk(source) if i.is_dir()): + new_directory = destination / source.name / directory.relative_to(source) + plan[directory] = new_directory + + # Find any files in this directory that are not yet in the DB and add them to the plan. + files = {i for i in walk(source) if i.is_file()} + if missing_files := files - set(plan.keys()): + for paths in group_files_by_stem(missing_files): + try: + # Only create the FileGroup if a primary_path can be found. + get_primary_file(paths) + file_group = FileGroup.from_paths(session, *paths) + add_files_to_refresh_count(1) + add_source_file_group_to_plan(file_group, source) + except NoPrimaryFile: + # No primary path for these paths, create FileGroups for each. + add_files_to_refresh_count(1) + for file in paths: + fg = FileGroup.from_paths(session, file) + add_source_file_group_to_plan(fg, source) + + # Sleep to catch cancel. + await asyncio.sleep(0) + else: + raise UnknownFile(f'Unknown path type, cannot move: {source}') + + # Sort plan by the deepest files first. + plan: OrderedDict = OrderedDict(sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True)) + + # Revert plan is built out as files are moved. + revert_plan = OrderedDict() + new_directories = set() + + def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path]): + """Apply the move plan to all the FileGroups. + + @warning: Cannot be cancelled!""" + # Move FileGroups in groups. + for chunk in chunks(plan_.items(), MOVE_CHUNK_SIZE): + old_paths = [i for i, j in chunk] + old_files, old_directories = partition(lambda i: i.is_file(), old_paths) + file_groups_ = session.query(FileGroup).filter(FileGroup.primary_path.in_(old_files)).all() + if len(file_groups_) != len(old_files): + # We ensured there are FileGroups while building the plan, maybe user deleted a file? + raise RuntimeError('Could not get all FileGroups for move') + + for file_group_ in file_groups_: + old_file = file_group_.primary_path + new_primary_path_ = plan_[old_file] + parent = new_primary_path_.parent + parent.mkdir(parents=True, exist_ok=True) + if parent not in new_directories: + new_directories.add(parent) + file_group_.move(new_primary_path_) + revert_plan[new_primary_path_] = old_file + for directory_ in old_directories: + delete_directory(directory_) + + existing_directories = {i[0] for i in session.query(Directory.path)} + missing_directories = new_directories - existing_directories + for directory_ in missing_directories: + session.add(Directory(path=directory_, name=directory_.name)) + + # Don't forget what has moved. + session.flush(file_groups_) + + with flags.refresh_discovery: + try: + do_plan(plan) + for source in sources: + if source.is_dir(): + delete_directory(source) + logger.info(f'Move execution completed') + except Exception as e: + logger.error(f'Move failed', exc_info=e) + new_directories = set() + # Move files back. Get copy because do_plan will change the revert plan. + do_plan(revert_plan.copy()) + # Delete the directories that were created. + directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True) + for path in directories: + if path.is_dir(): + delete_directory(path) + # Remove destination only if it did not exist at the start. + if destination.is_dir() and destination_existed is False: + delete_directory(destination) + raise + + return plan def delete_directory(directory: pathlib.Path, recursive: bool = False): @@ -1263,41 +1366,37 @@ def delete_directory(directory: pathlib.Path, recursive: bool = False): curs.execute(stmt, (str(directory),)) -async def move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]: +@optional_session +async def move(destination: pathlib.Path, *sources: pathlib.Path, session: Session = None) \ + -> OrderedDict[pathlib.Path, pathlib.Path]: """Moves a file or directory (recursively), preserving applied Tags. If the move fails at any point, the files will be returned to their previous locations. - Returns a list of tuples, the first Path in the tuple is the file's old location, the second Path is where the file - was moved. + Returns an OrderedDict, the key is the file's old location, the value is where the file was moved. """ - # Move the files first, if this fails it will revert itself and raise an error. - plan = _move(destination, *sources) + if not session: + raise RuntimeError('`session` is required') - # Files were moved, migrate any Tags. - plan_map = {old: new for old, new in plan} - old_file_paths = list(plan_map.keys()) - with get_db_session(commit=True) as session: - old_files: List[Tuple[FileGroup, TagFile]] = session.query(FileGroup, TagFile) \ - .filter(FileGroup.primary_path.in_(old_file_paths)) \ - .outerjoin(TagFile).all() - logger.info(f'Moving {len(old_files)} tagged files') - for old_file, _ in old_files: - try: - new_file_path = plan_map[old_file.primary_path] - logger.info(f'Moving tagged file {old_file.primary_path} to {new_file_path}') - old_file.move(new_file_path, move_files=False) - except KeyError as e: - # Got a duplicate? - logger.error('Error during tag migration', exc_info=e) - if PYTEST: - raise - session.commit() + with flags.refreshing: + from wrolpi.api_utils import api_app + api_app.shared_ctx.refresh['counted_files'] = 0 - if PYTEST: - await refresh_files([destination, *sources]) - else: - background_task(refresh_files([destination, *sources])) + # Move the files, if this fails it will revert itself and raise an error. + with timer('moving files', 'info'): + plan = await _move(destination, *sources, session=session) + session.commit() + + with flags.refresh_indexing: + await apply_indexers() + + with flags.refresh_modeling: + await apply_modelers() + + with flags.refresh_cleanup: + await apply_refresh_cleanup() + # Save tags now that files have been moved. + save_tags_config() return plan @@ -1320,25 +1419,40 @@ async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path: return new_path -async def rename_directory(directory: pathlib.Path, new_name: str) -> pathlib.Path: +async def rename_directory(directory: pathlib.Path, new_name: str, session: Session = None, send_events: bool = False) \ + -> pathlib.Path: """Rename a directory. This is done by moving all files into the new directory, and removing the old directory.""" + if not session: + raise RuntimeError('`session` is required') + new_directory = directory.with_name(new_name) if new_directory.exists(): raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.') # Move all paths into the new directory. paths = list(directory.iterdir()) - await move(new_directory, *paths) + try: + await move(new_directory, *paths, session=session) + if send_events: + Events.send_file_move_completed(f'Directory has been renamed: {directory}') + except Exception: + if send_events: + Events.send_file_move_failed(f'Directory rename has failed: {directory}') + raise # Remove the old directory. delete_directory(directory) return new_directory -async def rename(path: pathlib.Path, new_name: str) -> pathlib.Path: +@optional_session +async def rename(path: pathlib.Path, new_name: str, session: Session = None, send_events: bool = False) -> pathlib.Path: """Rename a directory or file. Preserve any tags.""" + if not session: + raise RuntimeError('`session` is required') + if path.is_dir(): - return await rename_directory(path, new_name) + return await rename_directory(path, new_name, session=session, send_events=send_events) return await rename_file(path, new_name) @@ -1399,26 +1513,36 @@ async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_nam paths = glob_shared_stem(pathlib.Path(file)) # Remove any ignored files. paths = remove_files_in_ignored_directories(paths) - if paths: - file_group = FileGroup.from_paths(session, *paths) - logger.debug(f'upsert_file: {file_group}') - # Re-index the contents of the file. - session.add(file_group) - session.flush([file_group, ]) - - try: - file_group.do_model(session) - except Exception as e: - logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e) - if PYTEST: - raise - - for tag_name in (tag_names or []): - if tag_name not in file_group.tag_names: - tag = Tag.get_by_name(tag_name) - file_group.add_tag(tag.id) - - session.commit() - return file_group - else: + if not paths: logger.warning('upsert_file called, but all files are in ignored directories!') + return + + for i in range(2): + file_group = FileGroup.from_paths(session, *paths) + # Re-index the contents of the file. + try: + session.flush([file_group, ]) + break + except IntegrityError: + # Another process inserted this FileGroup. + logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}') + session.rollback() + continue + else: + raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}') + + logger.debug(f'upsert_file: {file_group}') + try: + file_group.do_model(session) + except Exception as e: + logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e) + if PYTEST: + raise + + for tag_name in (tag_names or []): + if tag_name not in file_group.tag_names: + tag = Tag.get_by_name(tag_name) + file_group.add_tag(tag.id) + + session.commit() + return file_group diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 0ca41891..da848ae9 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -10,7 +10,7 @@ from sqlalchemy import types from sqlalchemy.orm import deferred, relationship, Session from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \ - get_relative_to_media_directory + get_relative_to_media_directory, unique_by_predicate from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime from wrolpi.db import optional_session from wrolpi.errors import FileGroupIsTagged, UnknownFile @@ -154,9 +154,8 @@ class FileGroup(ModelHelper, Base): """Add all `paths` to this FileGroup.files.""" from wrolpi.files.lib import get_mimetype new_files = list(self.files) if self.files else list() - for file in paths: - new_files.append(dict(path=file, mimetype=get_mimetype(file))) - self.files = new_files + new_files.extend([dict(path=path, mimetype=get_mimetype(path)) for path in paths]) + self.files = unique_by_predicate(new_files, lambda i: i['path']) def my_files(self, *mimetypes: str) -> List[dict]: """Return all files related to this group that match any of the provided mimetypes. @@ -225,7 +224,9 @@ class FileGroup(ModelHelper, Base): if session := Session.object_session(self): from wrolpi.downloader import download_manager - if self.url and add_to_skip_list is True: + if self.url and (download := download_manager.get_download(session, self.url)): + download.delete(add_to_skip_list) + elif self.url and add_to_skip_list is True: download_manager.add_to_skip_list(self.url) session.delete(self) @@ -254,25 +255,26 @@ class FileGroup(ModelHelper, Base): @classmethod def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup': """Create a new FileGroup which contains the provided file paths.""" - from wrolpi.files.lib import get_primary_file, get_mimetype, split_path_stem_and_suffix + from wrolpi.files.lib import get_primary_file, get_mimetype, get_unique_files_by_stem - unique_stems = {split_path_stem_and_suffix(i, full=True)[0] for i in paths} + unique_stems = get_unique_files_by_stem(paths) if len(unique_stems) > 1: raise RuntimeError(f'Refusing to create FileGroup with paths that do not share a stem.') existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all() + logger.debug(f'FileGroup.from_paths: {len(existing_groups)}') if len(existing_groups) == 0: # These paths have not been used previously, create a new FileGroup. file_group = FileGroup() primary_path = get_primary_file(paths) file_group.primary_path = primary_path file_group.append_files(*paths) - file_group.mimetype = get_mimetype(file_group.primary_path) session.add(file_group) elif len(existing_groups) == 1: # Found one FileGroup with these paths, no need to create a new FileGroup. file_group = existing_groups[0] - primary_path = file_group.primary_path + file_group.append_files(*paths) + primary_path = get_primary_file(file_group.my_paths()) else: # Multiple FileGroups contain these paths as primary. primary_path = get_primary_file(paths) @@ -284,8 +286,11 @@ class FileGroup(ModelHelper, Base): if not isinstance(primary_path, pathlib.Path): raise ValueError('Cannot create FileGroup without a primary path.') + file_group.primary_path = primary_path file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths)) file_group.size = sum(i.stat().st_size for i in paths) + file_group.mimetype = get_mimetype(file_group.primary_path) + logger.debug(f'FileGroup.from_paths: {file_group}') return file_group @@ -329,28 +334,44 @@ class FileGroup(ModelHelper, Base): self.files = collected_files - def move(self, new_primary_path: pathlib.Path, move_files: bool = True): + def move(self, new_primary_path: pathlib.Path): """Move all files in this group to a new location.""" - from wrolpi.files.lib import split_path_stem_and_suffix - if move_files and new_primary_path.exists(): + from wrolpi.files.lib import split_path_stem_and_suffix, glob_shared_stem + + if new_primary_path.exists(): raise FileExistsError(f'Cannot move {self} to {new_primary_path} because it already exists.') + # Adopt any new files when a FileGroup has multiple files. + if len(self.my_files()) > 1: + self.append_files(*glob_shared_stem(self.primary_path)) + new_name, _ = split_path_stem_and_suffix(new_primary_path, full=True) # Need a deepcopy because changes to self.files are ignored otherwise. new_files = copy.deepcopy(self.files) + # Ensure that all destination files do not yet exist before move. + for file in new_files: + _, suffix = split_path_stem_and_suffix(file['path']) + new_path = pathlib.Path(f'{new_name}{suffix}') + if new_path.exists(): + raise FileExistsError(f'Cannot move {self} to {new_path} because it already exists.') + + # Collect all files that exist and move them. + existing_files = list() for idx, file in enumerate(new_files): _, suffix = split_path_stem_and_suffix(file['path']) new_path = pathlib.Path(f'{new_name}{suffix}') - if move_files: + if pathlib.Path(file['path']).is_file(): shutil.move(file['path'], new_path) - new_files[idx]['path'] = new_path - self.files = new_files + existing_files.append({'path': new_path, 'mimetype': file['mimetype']}) + + logger.debug(f'Moved FileGroup: {self.primary_path} -> {new_primary_path}') + self.files = existing_files if self.title == self.primary_path.name: # Do not overwrite title from modeler. self.title = new_primary_path.name self.primary_path = new_primary_path # Need to re-index for self.data. - self.indexed = False + self.indexed = False if self.data else self.indexed # Flush the changes to the FileGroup. self.flush() @@ -384,11 +405,15 @@ class FileGroup(ModelHelper, Base): if model_class := self.get_model_class(): if model := model_class.get_by_path(self.primary_path, session): # Already modeled. + logger.debug(f'already modeled: {self} {model}') return model # Create new model (Video/Archive/Ebook). - return model_class.do_model(self, session) + model = model_class.do_model(self, session) + logger.debug(f'new model: {self} {model}') + return model # Index the file if no models are available. + logger.debug(f'no model class: {self}') self.do_index() self.indexed = True diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index a266b176..54a965d6 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -190,7 +190,7 @@ def test_files_search(test_session, test_client, make_files_structure, assert_fi @pytest.mark.asyncio async def test_files_search_any_tag(test_async_client, test_session, make_files_structure, tag_factory): - one, two = tag_factory(), tag_factory() + one, two = await tag_factory(), await tag_factory() files = [ 'foo.txt', 'foo bar.txt', @@ -299,34 +299,35 @@ def test_file_statistics(test_session, test_client, test_directory, example_pdf, } -def test_file_group_tag_by_primary_path(test_session, test_client, test_directory, example_singlefile, tag_factory, - insert_file_group): +@pytest.mark.asyncio +async def test_file_group_tag_by_primary_path(test_session, test_async_client, test_directory, example_singlefile, + tag_factory, insert_file_group): singlefile = FileGroup.from_paths(test_session, example_singlefile) - tag1 = tag_factory() - tag2 = tag_factory() + tag1 = await tag_factory() + tag2 = await tag_factory() test_session.commit() # FileGroup can be tagged with its primary_path. content = dict(file_group_primary_path=str(singlefile.primary_path.relative_to(test_directory)), tag_name=tag1.name) - request, response = test_client.post('/api/files/tag', content=json.dumps(content)) + request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content)) assert response.status_code == HTTPStatus.CREATED assert test_session.query(TagFile).count() == 1 # FileGroup can be tagged with its id. content = dict(file_group_id=singlefile.id, tag_name=tag2.name) - request, response = test_client.post('/api/files/tag', content=json.dumps(content)) + request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content)) assert response.status_code == HTTPStatus.CREATED assert test_session.query(TagFile).count() == 2 # FileGroup can be untagged with its primary_path. content = dict(file_group_id=singlefile.id, tag_id=tag1.id) - request, response = test_client.post('/api/files/untag', content=json.dumps(content)) + request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content)) assert response.status_code == HTTPStatus.NO_CONTENT assert test_session.query(TagFile).count() == 1 # FileGroup can be untagged with its id. content = dict(file_group_id=singlefile.id, tag_name=tag2.name) - request, response = test_client.post('/api/files/untag', content=json.dumps(content)) + request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content)) assert response.status_code == HTTPStatus.NO_CONTENT assert test_session.query(TagFile).count() == 0 @@ -468,11 +469,13 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f assert test_session.query(FileGroup).count() == 1 -def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form, tag_factory, - video_bytes): +@pytest.mark.asyncio +async def test_post_upload(test_session, test_async_client, test_directory, make_files_structure, make_multipart_form, + tag_factory, + video_bytes): """A file can be uploaded in chunks directly to the destination.""" make_files_structure(['uploads/']) - tag1, tag2 = tag_factory(), tag_factory() + tag1, tag2 = await tag_factory(), await tag_factory() part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:] forms = [ @@ -487,7 +490,7 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc ] body = make_multipart_form(forms) headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} - request, response = test_client.post('/api/files/upload', content=body, headers=headers) + request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers) assert response.status_code == HTTPStatus.OK, response.content.decode() output = test_directory / 'uploads/video.mp4' @@ -505,7 +508,7 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc dict(name='chunk', value=part2, filename='chunk'), ] body = make_multipart_form(forms) - request, response = test_client.post('/api/files/upload', content=body, headers=headers) + request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers) assert response.status_code == HTTPStatus.CREATED, response.content.decode() assert output.is_file() diff --git a/wrolpi/files/test/test_ebooks.py b/wrolpi/files/test/test_ebooks.py index 143cb92b..2655c5f1 100644 --- a/wrolpi/files/test/test_ebooks.py +++ b/wrolpi/files/test/test_ebooks.py @@ -137,7 +137,7 @@ async def test_discover_calibre_cover(test_session, test_directory, example_epub @pytest.mark.asyncio async def test_move_ebook(test_async_client, test_session, test_directory, example_epub, image_file, tag_factory): """An ebook is re-indexed when moved.""" - tag = tag_factory() + tag = await tag_factory() shutil.move(image_file, example_epub.with_suffix('.jpg')) await refresh_files() diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 9a79584f..4ca75dc4 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -82,7 +82,7 @@ async def test_delete_file_names(test_async_client, test_session, make_files_str }) foo_fg = FileGroup.from_paths(test_session, foo) foo1_fg = FileGroup.from_paths(test_session, foo1) - tag = tag_factory() + tag = await tag_factory() foo1_fg.add_tag(tag.id) test_session.commit() assert foo.is_file() @@ -106,7 +106,7 @@ async def test_delete_file_link(test_async_client, test_session, test_directory) @pytest.mark.asyncio async def test_delete_tagged(test_async_client, test_session, make_files_structure, tag_factory, video_bytes): """Cannot delete a file that has been tagged.""" - tag = tag_factory() + tag = await tag_factory() make_files_structure({'foo/bar.txt': 'asdf', 'foo/bar.mp4': video_bytes}) await lib.refresh_files() # Both files end up in a group. @@ -365,7 +365,7 @@ async def test_file_group_tag(test_session, make_files_structure, test_directory """A FileGroup can be tagged.""" make_files_structure(['foo.mp4']) await lib.refresh_discover_paths([test_directory, ], now()) - one = tag_factory() + one = await tag_factory() foo: FileGroup = test_session.query(FileGroup).one() foo.add_tag(one.id) @@ -401,7 +401,7 @@ async def test_refresh_many_files(test_async_client, test_session, make_files_st @pytest.mark.asyncio -async def test_refresh_cancel(test_session, make_files_structure, test_directory): +async def test_refresh_cancel(test_async_client, test_session, make_files_structure, test_directory): """Refresh tasks can be canceled.""" # Creat a lot of files so the refresh will take too long. make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)]) @@ -723,15 +723,16 @@ async def test_refresh_directories(test_session, test_directory, assert_director assert_directories({'foo', 'baz'}) -def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory, - video_bytes, srt_file3): +@pytest.mark.asyncio +async def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory, + video_bytes, srt_file3): """A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be migrated.""" vid, srt = make_files_structure({ 'vid.mp4': video_bytes, 'vid.srt': (PROJECT_DIR / 'test/example3.en.srt').read_text(), }) - one, two = tag_factory(), tag_factory() + one, two = await tag_factory(), await tag_factory() vid_group = FileGroup.from_paths(test_session, vid) srt_group = FileGroup.from_paths(test_session, srt) test_session.add_all([vid_group, srt_group]) @@ -774,10 +775,11 @@ async def test_move(test_async_client, test_session, test_directory, make_files_ }) foo = test_directory / 'foo' qux = test_directory / 'qux' + await lib.refresh_files() # mv foo qux - plan = await lib.move(qux, foo) - plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan] + plan = await lib.move(qux, foo, session=test_session) + plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan.items()] # The deepest files are moved first. assert plan == [('foo/bar/baz/archive.html', 'qux/foo/bar/baz/archive.html'), ('foo/bar/video.mp4', 'qux/foo/bar/video.mp4'), @@ -805,10 +807,16 @@ async def test_move_files(test_async_client, test_session, test_directory, make_ }) dest = test_directory / 'dest' - await lib.move(dest, one, two) + plan = await lib.move(dest, one, two, session=test_session) + assert list(plan.items()) == [ + (test_directory / 'foo/one.txt', test_directory / 'dest/one.txt'), + (test_directory / 'two.txt', test_directory / 'dest/two.txt'), + ] # one.txt is moved out of foo. - assert (dest / 'one.txt').read_text() == 'one' - assert (dest / 'two.txt').read_text() == 'two' + assert not (test_directory / 'foo/one.txt').exists(), 'one.txt is lingering in source' + assert not (test_directory / 'two.txt').exists(), 'two.txt is lingering in source' + assert (dest / 'one.txt').read_text() == 'one', 'one.txt was not moved to destination' + assert (dest / 'two.txt').read_text() == 'two', 'two.txt was not moved to destination' @pytest.mark.asyncio @@ -830,8 +838,14 @@ async def test_move_deep_directory(test_async_client, test_session, test_directo bar, qux = baz.parent, quux.parent dest = test_directory / 'deep/dest' - # mv foo/bar foo/qux foo/quuz.mp4 deep/dest - await lib.move(dest, bar, qux, quuz_mp4) + # mv foo/bar foo/qux foo/quuz.mp4 foo/quuz.txt deep/dest + plan = await lib.move(dest, bar, qux, quuz_mp4, quuz_txt, session=test_session) + assert list(plan.items()) == [ + (test_directory / 'foo/qux/quux.txt', test_directory / 'deep/dest/qux/quux.txt'), + (test_directory / 'foo/bar/baz.txt', test_directory / 'deep/dest/bar/baz.txt'), + (test_directory / 'foo/quuz.txt', test_directory / 'deep/dest/quuz.txt'), + (test_directory / 'foo/quuz.mp4', test_directory / 'deep/dest/quuz.mp4'), + ] # Unrelated files and directories are untouched. assert not (test_directory / 'deep/dest/foo').is_dir(), 'foo/ should not have been moved.' @@ -861,7 +875,7 @@ async def test_move_directory(test_async_client, test_session, test_directory, m assert_directories({'foo'}) bar = test_directory / 'bar' - await lib.rename(test_directory / 'foo', 'bar') + await lib.rename(test_directory / 'foo', 'bar', session=test_session) assert (bar / 'one.txt').read_text() == 'one' assert_directories({'bar'}) @@ -890,7 +904,7 @@ async def test_move_error(test_session, test_directory, make_files_structure, vi return shutil.move(*args, **kwargs) with mock.patch('wrolpi.files.lib.shutil.move', mock_shutil_move), pytest.raises(FileNotFoundError): - await lib.move(qux, foo) + await lib.move(qux, foo, session=test_session) # The move failed, the files should be moved back. # foo was not deleted. assert foo.is_dir() @@ -923,21 +937,19 @@ async def test_file_group_move(test_async_client, test_session, make_files_struc assert new_srt.read_text() == srt_text assert not video.is_file() assert not srt.is_file() - # Moved files must be re-indexed. - assert file_group.indexed is False @pytest.mark.asyncio async def test_move_tagged(test_async_client, test_session, test_directory, make_files_structure, tag_factory): """A FileGroup's tag is preserved when moved or renamed.""" - tag = tag_factory() + tag = await tag_factory() foo, bar, = make_files_structure({ 'foo/foo.txt': 'foo', 'foo/bar.txt': 'bar', }) await lib.refresh_files() bar_file_group, foo_file_group = test_session.query(FileGroup).order_by(FileGroup.primary_path) - foo_file_group.title = 'custom title' # Should not be overwritten. + bar_file_group.title = 'custom title' # Should not be overwritten. bar_file_group.add_tag(tag.id) test_session.commit() @@ -945,7 +957,7 @@ async def test_move_tagged(test_async_client, test_session, test_directory, make qux.mkdir() # Move both files into qux. The Tag should also be moved. - await lib.move(qux, bar, foo) + await lib.move(qux, bar, foo, session=test_session) new_foo = qux / 'foo.txt' new_bar = qux / 'bar.txt' # Files were moved. @@ -963,10 +975,10 @@ async def test_move_tagged(test_async_client, test_session, test_directory, make assert not foo_file_group.tag_files assert foo_file_group.primary_path.is_file() assert foo_file_group.primary_path == new_foo == foo_file_group.files[0]['path'] - assert foo_file_group.title == 'custom title', 'Custom title should not have been overwritten.' + assert bar_file_group.title == 'custom title', 'Custom title should not have been overwritten.' # Rename "bar.txt" to "baz.txt" - await lib.rename(new_bar, 'baz.txt') + await lib.rename(new_bar, 'baz.txt', session=test_session) baz = new_bar.with_name('baz.txt') assert baz.read_text() == 'bar' @@ -1111,3 +1123,21 @@ async def test_upsert_file_video_with_channel(test_async_client, test_session, t assert video.channel_id == channel.id, "Videos should be assigned to the Channel their files are in" assert video.video_path, 'Video file was upserted' assert video.info_json_path, 'Info json file not found near video file.' + + +@pytest.mark.asyncio +async def test_move_many_files(test_async_client, test_session, test_directory, make_files_structure): + make_files_structure([f'foo/{i}.txt' for i in range(10_000)]) + await lib.refresh_files() + + foo = test_directory / 'foo' + bar = test_directory / 'bar' + + with timer('test_move_many_files'): + # mv foo bar + await lib.move(bar, foo, session=test_session) + + assert not foo.is_dir() + assert bar.is_dir() + assert (bar / 'foo').is_dir() + assert (bar / 'foo/0.txt').is_file() diff --git a/wrolpi/files/test/test_models.py b/wrolpi/files/test/test_models.py new file mode 100644 index 00000000..78ee7944 --- /dev/null +++ b/wrolpi/files/test/test_models.py @@ -0,0 +1,21 @@ +import pytest + + +@pytest.mark.asyncio +async def test_file_group_move(test_async_client, test_session, test_directory, video_factory): + """Any new or renamed files are adopted during FileGroup.move.""" + video = video_factory(title='video', with_poster_ext='.webp') + test_session.commit() + + # Move one of the FileGroup's files. It should be moved as well. + video.poster_path.rename(video.poster_path.with_suffix('.jpg')) + + # Move the FilGroup. + new_primary_path = video.video_path.with_name('new name') + video.file_group.move(new_primary_path) + + # New file is discovered and adopted, missing file is removed. + assert video.file_group.my_paths() == [ + (test_directory / 'videos/NO CHANNEL/new name.jpg'), + (test_directory / 'videos/NO CHANNEL/new name.mp4'), + ] diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index e0ffb19b..4c261d47 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -23,7 +23,7 @@ from wrolpi.admin import HotspotStatus from wrolpi.api_utils import json_response, json_error_handler, api_app from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_media_directory, \ wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, \ - set_global_log_level, get_relative_to_media_directory + set_global_log_level, get_relative_to_media_directory, search_other_estimates from wrolpi.dates import now from wrolpi.downloader import download_manager from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \ @@ -31,6 +31,7 @@ from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \ from wrolpi.events import get_events, Events from wrolpi.files import files_bp from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count +from wrolpi.tags import Tag from wrolpi.vars import DOCKERIZED, IS_RPI, IS_RPI4, IS_RPI5, API_HOST, API_PORT, API_WORKERS, API_DEBUG, \ API_ACCESS_LOG, truthy_arg, API_AUTO_RELOAD from wrolpi.version import __version__ @@ -118,10 +119,16 @@ async def index(_): @openapi.description('Echo whatever is sent to this.') @openapi.response(HTTPStatus.OK, schema.EchoResponse) async def echo(request: Request): + try: + request_json = request.json + except Exception as e: + logger.error('Failed to parse request JSON', exc_info=e) + request_json = None + request_headers = dict(request.headers) ret = dict( form=request.form, - headers=dict(request.headers), - json=request.json, + headers=request_headers, + json=request_json, method=request.method, args=request.args, ) @@ -137,7 +144,7 @@ def get_settings(_: Request): ignored_directories = [get_relative_to_media_directory(i) for i in config.ignored_directories] settings = { - 'archive_directory': config.archive_destination, + 'archive_destination': config.archive_destination, 'download_manager_disabled': download_manager.is_disabled, 'download_manager_stopped': download_manager.is_stopped, 'download_on_startup': config.download_on_startup, @@ -150,15 +157,15 @@ def get_settings(_: Request): 'ignore_outdated_zims': config.ignore_outdated_zims, 'ignored_directories': ignored_directories, 'log_level': api_app.shared_ctx.log_level.value, - 'map_directory': config.map_destination, + 'map_destination': config.map_destination, 'nav_color': config.nav_color, 'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative. 'throttle_on_startup': config.throttle_on_startup, 'throttle_status': admin.throttle_status().name, 'version': __version__, - 'videos_directory': config.videos_destination, + 'videos_destination': config.videos_destination, 'wrol_mode': config.wrol_mode, - 'zims_directory': config.zims_destination, + 'zims_destination': config.zims_destination, } return json_response(settings) @@ -482,7 +489,7 @@ async def get_tags_request(_: Request): summary='Create or update a Tag', ) async def post_tag(_: Request, body: schema.TagRequest, tag_id: int = None): - tags.upsert_tag(body.name, body.color, tag_id) + await tags.upsert_tag(body.name, body.color, tag_id) if tag_id: return response.empty(HTTPStatus.OK) else: @@ -491,7 +498,7 @@ async def post_tag(_: Request, body: schema.TagRequest, tag_id: int = None): @api_bp.delete('/tag/') async def delete_tag_request(_: Request, tag_id: int): - tags.delete_tag(tag_id) + Tag.find_by_id(tag_id).delete() return response.empty() @@ -594,4 +601,13 @@ async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimate return json_response(ret) +@api_bp.post('/search_other_estimates') +@validate(json=schema.SearchOtherEstimateRequest) +async def post_search_other_estimates(_: Request, body: schema.SearchOtherEstimateRequest): + """Used by the Global search to suggest FileGroup count to the user.""" + others = await search_other_estimates(body.tag_names) + ret = dict(others=others) + return json_response(ret) + + api_app.error_handler.add(Exception, json_error_handler) diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 2fadb3c5..00d18e1a 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -206,3 +206,8 @@ class SearchFileEstimateRequest: def __post_init__(self): if self.any_tag and self.tag_names: raise ValidationError('Cannot search for any tag, and list of tags.') + + +@dataclass +class SearchOtherEstimateRequest: + tag_names: List[str] = field(default_factory=lambda: list()) diff --git a/wrolpi/tags.py b/wrolpi/tags.py index a824d538..0d60bfff 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -54,7 +54,9 @@ class Tag(ModelHelper, Base): color = Column(String) tag_files: List[TagFile] = relationship('TagFile', back_populates='tag', cascade='all') - tag_zim_entries: List[TagFile] = relationship('TagZimEntry', back_populates='tag', cascade='all') + tag_zim_entries: List = relationship('TagZimEntry', back_populates='tag', cascade='all') + channels: List = relationship('Channel', primaryjoin='Tag.id==Channel.tag_id', back_populates='tag', + cascade='all') def __repr__(self): name = self.name @@ -148,27 +150,71 @@ class Tag(ModelHelper, Base): def has_relations(self) -> bool: """Returns True if this Tag has been used with any FileGroups or Zim Entries.""" - return bool(self.tag_files or self.tag_zim_entries) + return bool(any(self.tag_files) or any(self.tag_zim_entries) or any(self.channels)) - def update_tag(self, name: str, color: str, session: Session): + async def update_tag(self, name: str, color: str | None, session: Session): """Change name/color of tag. Ensures safeness of new values, checks for conflicts.""" if ',' in name: raise InvalidTag('Tag name cannot have comma') - if not is_valid_hex_color(color): + if color and not is_valid_hex_color(color): raise InvalidTag('Tag color is invalid') - # Replace forward-slash (linux directories) with unicode Big Solidus (U+29F8) + # Replace forward-slash (linux directories) with Unicode Big Solidus (U+29F8) name = name.replace('/', '⧸') name = INVALID_FILE_CHARS.sub('', name) - if Tag.get_by_name(name, session): + if (other := self.get_by_name(name, session)) and other.id != self.id: raise InvalidTag('Tag name already taken') if flags.refreshing.is_set(): raise RefreshConflict('Refusing to rename tag while file refresh is in progress') + old_name = self.name self.name = name - self.color = color + self.color = color or self.color + if Session.object_session(self): + self.flush() + + from modules.videos.models import Channel + from modules.videos.errors import ChannelDirectoryConflict + + async def _(): + for channel in self.channels: + channel: Channel + possible_directory = channel.format_directory(old_name) + if channel.directory == possible_directory: + # Channel is in this Tag's old directory, move the Channel to the new directory. + new_directory = channel.format_directory(name) + try: + new_directory.mkdir(parents=True) + except FileExistsError: + raise ChannelDirectoryConflict(f'Channel directory already exists: {new_directory}') + + # Move the files of the Channel. + await channel.move_channel(new_directory, session) + else: + msg = f"Not moving Channel because it is not in this Tag's old directory:" \ + f" {channel} {possible_directory} {self}" + logger.warning(msg) + + if PYTEST: + await _() + else: + background_task(_()) + + def delete(self): + """Deletes this Tag, if it is unused. + + @warning: Commits and saves configs""" + if self.has_relations(): + raise UsedTag(f'Cannot delete {self.name} it is used') + + session = Session.object_session(self) + session.delete(self) + session.commit() + + save_tags_config(session) + sync_tags_directory(session) class TagsConfig(ConfigFile): @@ -394,16 +440,17 @@ def get_tags() -> List[dict]: @optional_session @run_after(save_tags_config) -def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = None) -> Tag: +async def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = None) -> Tag: if tag_id: tag = Tag.find_by_id(tag_id, session) - tag.update_tag(name, color, session) - tag.flush() + await tag.update_tag(name, color, session) else: tag = Tag() - tag.update_tag(name, color, session) + await tag.update_tag(name, color, session) session.add(tag) + tag.flush() + try: session.commit() except IntegrityError as e: @@ -411,26 +458,12 @@ def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = Non session.rollback() raise InvalidTag(f'Name already taken') from e - save_tags_config() - sync_tags_directory() + save_tags_config(session) + sync_tags_directory(session) return tag -@optional_session -def delete_tag(tag_id: int, session: Session = None): - tag = Tag.find_by_id(tag_id, session) - - if tag.has_relations(): - raise UsedTag(f'Cannot delete {tag.name} it is used') - - session.delete(tag) - session.commit() - - save_tags_config() - sync_tags_directory() - - @optional_session def import_tags_config(session: Session = None): """Reads the Tags and TagFiles from the config file, upserts them in the DB.""" diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index f3db63b6..8dbb5e2c 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -901,3 +901,18 @@ def test_html_screenshot(singlefile_contents_factory): ]) def test_is_valid_hex_color(color, expected): assert common.is_valid_hex_color(color) == expected + + +@pytest.mark.parametrize('iterable,predicate,expected', [ + ((1, 2, 2, 3, 4), lambda i: i, (1, 2, 3, 4)), + ([1, 1, 2, 3, 3, 3, 3, 4], None, [1, 2, 3, 4]), + ([1, 1, 2, 2, 3, 4, 5, 6, 7, 8], lambda i: i % 2, [1, 2]), + (['apple', 'banana', 'pear', 'kiwi', 'plum', 'peach'], lambda i: len(i), ['apple', 'banana', 'pear']), + ( + [pathlib.Path('foo.txt'), pathlib.Path('foo.mp4'), pathlib.Path('bar.txt')], + lambda i: i.stem, + [pathlib.Path('foo.txt'), pathlib.Path('bar.txt')] + ), +]) +def test_unique_by_predicate(iterable, predicate, expected): + assert common.unique_by_predicate(iterable, predicate) == expected diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index b1148227..0368ab95 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -76,7 +76,7 @@ async def test_create_downloads(test_async_client, test_session, test_download_m @pytest.mark.asyncio async def test_recreate_download(test_session, test_download_manager, test_downloader, tag_factory): """Settings are preserved when a Download is restarted.""" - tag = tag_factory() + tag = await tag_factory() settings = {'tag_names': [tag.name, ]} # Create download with settings initially. diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 91c0aa82..6a501df0 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -392,7 +392,7 @@ async def test_get_status(test_async_client, test_session): @pytest.mark.asyncio async def test_download_crud(test_session, test_async_client, test_download_manager_config, tag_factory): """Test creating once-downloads and recurring downloads.""" - tag1, tag2 = tag_factory(), tag_factory() + tag1, tag2 = await tag_factory(), await tag_factory() async def dispatch_downloads(*a, **kw): pass @@ -567,7 +567,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_) assert response_.status_code == HTTPStatus.OK - tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory() + tag1, tag2, tag3 = await tag_factory(), await tag_factory(), await tag_factory() archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ]) archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ]) video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name]) @@ -629,7 +629,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa async def test_search_file_estimates_any_tag(test_session, test_async_client, archive_factory, tag_factory, video_factory): """File estimates can be filtered by any_tag.""" - tag1 = tag_factory() + tag1 = await tag_factory() archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ]) archive_factory(domain='bar.com', contents='contents of bar bunny') test_session.commit() @@ -706,6 +706,12 @@ def test_recursive_errors(): @pytest.mark.asyncio async def test_settings_special_directories(test_async_client, test_config): """Maintainer can change special directories.""" + request, response = await test_async_client.get('/api/settings') + assert response.status_code == HTTPStatus.OK + assert response.json['archive_destination'] == 'archive/%(domain)s' + assert response.json['map_destination'] == 'map' + assert response.json['videos_destination'] == 'videos/%(channel_tag)s/%(channel_name)s' + assert response.json['zims_destination'] == 'zims' data = {'archive_destination': 'archives'} request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) @@ -731,3 +737,23 @@ async def test_settings_special_directories(test_async_client, test_config): request, response = await test_async_client.patch('/api/settings', content=json.dumps(data)) assert response.status_code == HTTPStatus.NO_CONTENT assert get_wrolpi_config().archive_destination == 'archive/%(domain)s' + + +@pytest.mark.asyncio +async def test_search_other_estimates(test_async_client, test_session, channel_factory, tag_factory): + # Can search without Tags or Channels. + body = dict(tag_names=[]) + request, response = await test_async_client.post('/api/search_other_estimates', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['others']['channel_count'] == 0, 'Only one Channel is tagged.' + + tag = await tag_factory() + channel1 = channel_factory(tag_name=tag.name) + channel2 = channel_factory() + test_session.commit() + + # One Channel is tagged. + body = dict(tag_names=[tag.name, ]) + request, response = await test_async_client.post('/api/search_other_estimates', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.' diff --git a/wrolpi/test/test_tags.py b/wrolpi/test/test_tags.py index 2356bee6..3e3fa547 100644 --- a/wrolpi/test/test_tags.py +++ b/wrolpi/test/test_tags.py @@ -6,7 +6,7 @@ import yaml from wrolpi import tags from wrolpi.common import is_hardlinked, walk -from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, FileGroupAlreadyTagged +from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, FileGroupAlreadyTagged, UsedTag from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup from wrolpi.tags import TagFile, Tag @@ -15,8 +15,8 @@ from wrolpi.tags import TagFile, Tag @pytest.mark.asyncio async def test_tags_file_group_json(test_async_client, test_session, make_files_structure, tag_factory, example_pdf): """The tags of a file are returned in its JSON.""" - tag_one = tag_factory() - tag_two = tag_factory() + tag_one = await tag_factory() + tag_two = await tag_factory() await files_lib.refresh_files() file_group: FileGroup = test_session.query(FileGroup).one() @@ -37,7 +37,7 @@ async def test_tags_file_group_json(test_async_client, test_session, make_files_ @pytest.mark.asyncio async def test_tags_model(test_async_client, test_session, make_files_structure, tag_factory, example_pdf): """Can get Tag using class methods.""" - tag1 = tag_factory() + tag1 = await tag_factory() assert tag1.__json__() assert Tag.find_by_id(tag1.id) @@ -63,7 +63,7 @@ async def test_tags_file_group(test_async_client, test_session, make_files_struc await files_lib.refresh_files() video_group: FileGroup = test_session.query(FileGroup).one() - tag_one = tag_factory() + tag_one = await tag_factory() assert len(tag_one.tag_files) == 0 Tag.tag_file_group(video_group.id, tag_one.id) @@ -71,7 +71,7 @@ async def test_tags_file_group(test_async_client, test_session, make_files_struc assert len(tag_one.tag_files) == 1 assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', ] - tag_two = tag_factory() + tag_two = await tag_factory() Tag.tag_file_group(video_group.id, tag_two.id) test_session.commit() assert len(tag_one.tag_files) == 1 @@ -95,8 +95,8 @@ async def test_tags_config_file(test_async_client, test_session, test_directory, await files_lib.refresh_files() pdf: FileGroup = FileGroup.get_by_path(example_pdf, test_session) video: FileGroup = FileGroup.get_by_path(video_file, test_session) - tag1 = tag_factory() - tag2 = tag_factory() + tag1 = await tag_factory() + tag2 = await tag_factory() test_session.commit() tags.save_tags_config(test_session) @@ -172,11 +172,17 @@ async def test_tags_crud(test_async_client, test_session, example_pdf, assert_ta dict(name='ガーデン', color='#000000', id=1, file_group_count=1, zim_entry_count=0)] assert_tags_config(tags={'ガーデン': {'color': '#000000'}}) + # Tag color can be changed + content = dict(name='ガーデン', color='#ffffff') + request, response = await test_async_client.post('/api/tag/1', content=json.dumps(content)) + assert response.status_code == HTTPStatus.OK + assert Tag.find_by_name('ガーデン').color == '#ffffff' + # Conflicting names return an error. content = dict(name='ガーデン', color='#111111') request, response = await test_async_client.post('/api/tag', content=json.dumps(content)) assert response.status_code == HTTPStatus.BAD_REQUEST - assert_tags_config(tags={'ガーデン': {'color': '#000000'}}) + assert_tags_config(tags={'ガーデン': {'color': '#ffffff'}}) # Cannot delete Tag that is used. request, response = await test_async_client.delete('/api/tag/1') @@ -197,7 +203,7 @@ async def test_tags_crud(test_async_client, test_session, example_pdf, assert_ta async def test_delete_tagged_file(test_session, example_pdf, tag_factory): """You cannot delete a FileGroup if it is tagged.""" await files_lib.refresh_files() - tag = tag_factory() + tag = await tag_factory() pdf: FileGroup = test_session.query(FileGroup).one() pdf.add_tag(tag.id) @@ -277,7 +283,7 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag from modules.zim import lib as zim_lib # Create some tags, use tag1. - tag1, tag2, tag3, tag4 = tag_factory(), tag_factory(), tag_factory(), tag_factory() + tag1, tag2, tag3, tag4 = await tag_factory(), await tag_factory(), await tag_factory(), await tag_factory() # use tag1 for FileGroup. foo, = make_files_structure({'foo': 'text'}) foo = FileGroup.from_paths(test_session, foo) @@ -324,16 +330,13 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag @pytest.mark.asyncio async def test_invalid_tag(test_async_client, test_session, test_directory): with pytest.raises(InvalidTag): - tags.upsert_tag('cannot use comma: ,', '#fff') + await tags.upsert_tag('cannot use comma: ,', '#fff') with pytest.raises(InvalidTag): - tags.upsert_tag('invalid color', 'foo') + await tags.upsert_tag('invalid color', 'foo') with pytest.raises(UnknownTag): - tags.upsert_tag('Tag ID does not exist', '#fff', 1) - - with pytest.raises(UnknownTag): - tags.delete_tag(1) + await tags.upsert_tag('Tag ID does not exist', '#fff', 1) @pytest.mark.asyncio @@ -347,7 +350,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t (test_directory / 'tags/cannot be deleted').mkdir(parents=True) (test_directory / 'tags/cannot be deleted/because of this file').touch() - tag1, tag2 = tag_factory('First Aid'), tag_factory('Special/name') + tag1, tag2 = await tag_factory('First Aid'), await tag_factory('Special/name') vid1 = video_factory(with_video_file=test_directory / 'vid1.mp4', with_caption_file=True, with_info_json=True, with_poster_ext='jpg') vid2 = video_factory(with_video_file=test_directory / 'vid2.mp4', with_info_json=True) @@ -443,7 +446,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t async def test_update_tag(test_async_client, test_session, test_directory, video_factory, tag_factory): """Linked files in the Tag Directory are moved when the tag is renamed.""" # Create tagged Video. - tag = tag_factory() + tag = await tag_factory() video_factory(title='video', tag_names=[tag.name]) test_session.commit() @@ -453,10 +456,33 @@ async def test_update_tag(test_async_client, test_session, test_directory, video assert (test_directory / 'tags/one/video.mp4').is_file() # Invalid characters are replaced during rename. - tags.upsert_tag('new/name%', tag.color, tag.id) + await tags.upsert_tag('new/name%', tag.color, tag.id) # Video file was moved. assert (test_directory / 'tags').is_dir() assert (test_directory / 'videos/NO CHANNEL/video.mp4').is_file() assert not (test_directory / 'tags/one/video.mp4').exists() assert (test_directory / 'tags/new⧸name/video.mp4').is_file() + + +@pytest.mark.asyncio +async def test_tag_rename_with_channel(test_async_client, test_session, test_directory, video_factory, tag_factory, + channel_factory): + tag = await tag_factory() + channel = channel_factory(name='Channel Name', tag_name=tag.name) + video_factory(title='video', channel_id=channel.id) + test_session.commit() + + assert (test_directory / 'tags').is_dir() + assert (test_directory / 'videos/one/Channel Name').is_dir() + assert (test_directory / 'videos/one/Channel Name/video.mp4').is_file() + + await tag.update_tag('New Tag Name', None, session=test_session) + + assert (test_directory / 'tags').is_dir() + assert (test_directory / 'videos/New Tag Name/Channel Name').is_dir() + assert (test_directory / 'videos/New Tag Name/Channel Name/video.mp4').is_file() + + # Cannot delete Tag when used by Channel. This prevents the need to move Channel directories when Tags are deleted. + with pytest.raises(UsedTag): + tag.delete()