diff --git a/app/src/components/Download.js b/app/src/components/Download.js
index 73197413..1d052944 100644
--- a/app/src/components/Download.js
+++ b/app/src/components/Download.js
@@ -43,10 +43,8 @@ class Downloader extends React.Component {
this.setState({pending: true, submitted: false});
try {
let response = await postDownload(urls, downloader, null, null, null, destination, tagNames);
- if (response.status === 204) {
+ if (response.ok) {
this.setState({urls: '', pending: false, submitted: true});
- } else {
- toast({type: 'error', title: 'Error', description: 'Failed to create download!'});
}
} finally {
this.setState({pending: false});
@@ -216,8 +214,7 @@ class ChannelDownload extends React.Component {
if (response.status === 204) {
this.setState({pending: false, disabled: false, success: true, url: '', ready: false});
} else {
- const content = await response.json();
- let error = content.message || null;
+ let error = (await response.json()).message || null;
this.setState({pending: false, disabled: false, success: false, error});
}
}
@@ -355,8 +352,7 @@ class RSSDownload extends ChannelDownload {
if (response.status === 204) {
this.setState({pending: false, disabled: false, success: true, url: '', ready: false});
} else {
- const content = await response.json();
- let error = content.message || null;
+ let error = (await response.json()).message || null;
this.setState({pending: false, disabled: false, success: false, error});
}
}
@@ -487,15 +483,8 @@ class ScrapeDownloader extends Downloader {
suffix,
max_pages,
);
- if (response.status === 204) {
+ if (response.ok) {
this.setState({urls: '', pending: false, submitted: true});
- } else {
- const content = await response.json();
- let error = 'Failed to create download!';
- if (content && content['error']) {
- error = content['error'];
- }
- toast({type: 'error', title: 'Error', description: error});
}
} finally {
this.setState({pending: false});
diff --git a/app/src/components/Files.js b/app/src/components/Files.js
index 39a4c126..50eb551e 100644
--- a/app/src/components/Files.js
+++ b/app/src/components/Files.js
@@ -20,7 +20,9 @@ import {
ErrorMessage,
ExternalCardLink,
FileIcon,
+ FILES_MEDIA_URI,
findPosterPath,
+ HandPointMessage,
humanFileSize,
isoDatetimeToString,
mimetypeColor,
@@ -583,6 +585,10 @@ function FilesPage() {
return <>
+
+
+ You can also view your media directory at {FILES_MEDIA_URI}
+
>;
}
diff --git a/app/src/components/Inventory.js b/app/src/components/Inventory.js
index 634cfcb2..4ad400e7 100644
--- a/app/src/components/Inventory.js
+++ b/app/src/components/Inventory.js
@@ -504,35 +504,19 @@ class EditInventory extends React.Component {
handleSubmit = async (inventory, closeCallback) => {
let inventoryId = inventory.id;
- delete inventory['id'];
let response = await updateInventory(inventoryId, inventory);
-
- if (response.status === 204) {
+ if (response.ok) {
await this.props.success();
closeCallback();
- } else {
- toast({
- type: 'error',
- header: 'Failed to save inventory!',
- description: 'Server responded with an error.',
- time: 5000,
- })
}
}
handleDelete = async (inventory, closeCallback) => {
let response = await deleteInventory(inventory.id);
- if (response.status === 204) {
+ if (response.ok) {
await this.props.success();
closeCallback();
- } else {
- toast({
- type: 'error',
- header: 'Failed to delete inventory!',
- description: 'Server responded with an error.',
- time: 5000,
- })
}
}
@@ -554,16 +538,9 @@ class NewInventory extends React.Component {
delete inventory.id;
let response = await saveInventory(inventory);
- if (response.status === 201) {
+ if (response.ok) {
await this.props.success();
closeCallback();
- } else {
- toast({
- type: 'error',
- header: 'Failed to save inventory!',
- description: 'Server responded with an error.',
- time: 5000,
- })
}
}
@@ -953,18 +930,11 @@ class InventoryAddList extends React.Component {
expiration_date: this.state.expiration_date,
};
let response = await saveItem(this.state.inventory.id, item);
- if (response.status === 204) {
+ if (response.ok) {
await this.clearInputs();
await this.fetchItems();
await this.fetchCategories();
await this.fetchBrands();
- } else {
- toast({
- type: 'error',
- header: 'Failed to save item!',
- description: 'Server responded with an error.',
- time: 5000,
- })
}
}
diff --git a/app/src/components/Search.js b/app/src/components/Search.js
index dd6dc00f..31f1db26 100644
--- a/app/src/components/Search.js
+++ b/app/src/components/Search.js
@@ -1,15 +1,24 @@
import React, {useState} from "react";
-import {Route, Routes, useNavigate} from "react-router-dom";
+import {Link, Route, Routes, useNavigate} from "react-router-dom";
import {FilesSearchView} from "./Files";
-import {useLatestRequest, usePages, useSearchDate, useSearchFilter} from "../hooks/customHooks";
+import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchFilter} from "../hooks/customHooks";
import {ZimSearchView} from "./Zim";
-import {searchEstimateFiles, searchEstimateZims, searchSuggestions} from "../api";
+import {searchEstimateFiles, searchEstimateOthers, searchEstimateZims, searchSuggestions} from "../api";
import {filterToMimetypes, fuzzyMatch, normalizeEstimate, SearchResultsInput, TabLinks} from "./Common";
import _ from "lodash";
import {TagsContext} from "../Tags";
-import {Button as SButton, Header as SHeader, Label} from "semantic-ui-react";
-import {Modal, ModalContent} from "./Theme";
-import {QueryContext} from "../contexts/contexts";
+import {
+ AccordionContent,
+ AccordionTitle,
+ Button as SButton,
+ Grid,
+ GridColumn,
+ GridRow,
+ Header as SHeader,
+ Label
+} from "semantic-ui-react";
+import {Accordion, Header, Icon, Loader, Modal, ModalContent, Segment} from "./Theme";
+import {QueryContext, ThemeContext} from "../contexts/contexts";
const SUGGESTED_APPS = [
{location: '/admin', title: 'Downloads', description: 'View and control your downloads'},
@@ -128,6 +137,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
const {data: filesData, sendRequest: sendFilesRequest, loading: filesLoading} = useLatestRequest(500);
// Zims are slow, so they are separate.
const {data: zimData, sendRequest: sendZimRequest, loading: zimLoading} = useLatestRequest(500);
+ const {data: otherData, sendRequest: sendOtherRequest, loading: otherLoading} = useLatestRequest(500);
React.useEffect(() => {
if (searchStr || (tagNames && tagNames.length > 0)) {
@@ -139,6 +149,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
}
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
+ sendOtherRequest(async () => await searchEstimateOthers(tagNames));
}
}, [
searchStr,
@@ -179,6 +190,14 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
}
}, [setSuggestions, zimData]);
+ React.useEffect(() => {
+ if (!_.isEmpty(otherData)) {
+ setSuggestions((prevState) => {
+ return {...prevState, otherEstimates: otherData.others}
+ });
+ }
+ }, [setSuggestions, otherData]);
+
return {suggestions, loading: generalLoading || zimLoading || filesLoading}
}
@@ -269,6 +288,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
}
}
+ const otherSum = suggestions.otherEstimates ? _.sum(Object.values(suggestions.otherEstimates)) : 0;
+
const matchingApps = SUGGESTED_APPS.filter(i =>
i.title.toLowerCase().includes(lowerSearchStr)
|| fuzzyMatch(i.title.toLowerCase(), lowerSearchStr)
@@ -283,6 +304,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
setSuggestionsSums({
fileGroups: newSuggestions.fileGroups,
zims: zimSum,
+ otherSum: otherSum,
channels: newSuggestions.channels.length,
domains: newSuggestions.domains.length,
tags: matchingTags?.length,
@@ -342,10 +364,12 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
const filesTabName =
Files ;
const zimsTabName =
Zims ;
+ const othersTabName =
Other ;
const links = [
{text: filesTabName, to: '/search', key: 'filesSearch_', end: true},
{text: zimsTabName, to: '/search/zim', key: 'zimsSearch'},
+ {text: othersTabName, to: '/search/other', key: 'othersSearch'},
];
return
@@ -353,6 +377,7 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
}/>
}/>
+ }/>
}
@@ -403,3 +428,58 @@ export function SearchIconButton() {
}
+
+function SearchChannelPreview({channel}) {
+ const {t} = React.useContext(ThemeContext);
+
+ return
+
+
+ {channel.name}
+
+
+
+}
+
+function OtherSearchView({loading}) {
+ const {searchParams} = React.useContext(QueryContext);
+ const [activeIndex, setActiveIndex] = React.useState(0);
+ const activeTags = searchParams.getAll('tag');
+ const {channels, loading: channelsLoading} = useSearchChannels(activeTags);
+
+ const handleClick = (newIndex) => {
+ console.log('newIndex', newIndex, 'activeIndex', activeIndex);
+ setActiveIndex(activeIndex === newIndex ? null : newIndex);
+ }
+
+ if (loading || channelsLoading) {
+ return
+
+
+ }
+
+ const channelsAccordion =
+ handleClick(0)}
+ >
+
+
+ Channels
+
+
+
+
+
+ {!_.isEmpty(channels) ?
+ channels.map(i => )
+ : 'No Channels'}
+
+
+ ;
+
+ return
+ {channelsAccordion}
+
+}
\ No newline at end of file
diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js
index 14e1955f..d2bbb5f6 100644
--- a/app/src/components/Videos.js
+++ b/app/src/components/Videos.js
@@ -243,6 +243,7 @@ function VideosStatistics() {
{key: 'month', label: 'Downloads Past Month'},
{key: 'year', label: 'Downloads Past Year'},
{key: 'sum_duration', label: 'Total Duration'},
+ {key: 'censored_videos', label: 'Censored Videos'},
];
const commentsNames = [
{key: 'have_comments', label: 'Have Comments'},
@@ -255,6 +256,7 @@ function VideosStatistics() {
];
const channelNames = [
{key: 'channels', label: 'Channels'},
+ {key: 'tagged_channels', label: 'Tagged Channels'},
];
const buildSegment = (title, names, stats) => {
diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js
index 45afe812..ebf732f1 100644
--- a/app/src/components/admin/Downloads.js
+++ b/app/src/components/admin/Downloads.js
@@ -255,18 +255,7 @@ class RecurringDownloadRow extends React.Component {
handleRestart = async () => {
const {id} = this.props;
try {
- const response = await restartDownload(id);
- if (response.status !== 204) {
- throw Error('Unable to restart download');
- }
- } catch (e) {
- toast({
- type: 'error',
- title: 'Error',
- description: 'Unable to restart download',
- time: 5000,
- })
- throw e;
+ await restartDownload(id);
} finally {
if (this.props.fetchDownloads) {
this.props.fetchDownloads();
diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js
index 22d0c37d..351de605 100644
--- a/app/src/components/admin/Settings.js
+++ b/app/src/components/admin/Settings.js
@@ -181,7 +181,7 @@ export class SettingsPage extends React.Component {
qrOpen: false,
ready: false,
- archive_directory: null,
+ archive_destination: null,
download_on_startup: null,
download_timeout: null,
hotspot_device: null,
@@ -190,12 +190,12 @@ export class SettingsPage extends React.Component {
hotspot_status: null,
ignore_outdated_zims: null,
log_level: null,
- map_directory: null,
+ map_destination: null,
navColor: null,
throttle_on_startup: null,
throttle_status: null,
- videos_directory: null,
- zims_directory: null,
+ videos_destination: null,
+ zims_destination: null,
}
this.handleSubmit = this.handleSubmit.bind(this);
@@ -207,7 +207,7 @@ export class SettingsPage extends React.Component {
const settings = await getSettings();
this.setState({
ready: true,
- archive_directory: settings.archive_directory,
+ archive_destination: settings.archive_destination,
disabled: settings.wrol_mode,
download_on_startup: settings.download_on_startup,
download_timeout: settings.download_timeout || '',
@@ -217,12 +217,12 @@ export class SettingsPage extends React.Component {
hotspot_status: settings.hotspot_status,
ignore_outdated_zims: settings.ignore_outdated_zims,
log_level: fromApiLogLevel(settings.log_level),
- map_directory: settings.map_directory,
+ map_destination: settings.map_destination,
navColor: settings.nav_color || 'violet',
throttle_on_startup: settings.throttle_on_startup,
throttle_status: settings.throttle_status,
- videos_directory: settings.videos_directory,
- zims_directory: settings.zims_directory,
+ videos_destination: settings.videos_destination,
+ zims_destination: settings.zims_destination,
}, this.handleHotspotChange);
} catch (e) {
console.error(e);
@@ -233,7 +233,7 @@ export class SettingsPage extends React.Component {
async handleSubmit(callback) {
this.setState({disabled: true, pending: true});
let settings = {
- archive_directory: this.state.archive_directory,
+ archive_destination: this.state.archive_destination,
download_on_startup: this.state.download_on_startup,
download_timeout: this.state.download_timeout ? parseInt(this.state.download_timeout) : 0,
hotspot_device: this.state.hotspot_device,
@@ -241,11 +241,11 @@ export class SettingsPage extends React.Component {
hotspot_ssid: this.state.hotspot_ssid,
ignore_outdated_zims: this.state.ignore_outdated_zims,
log_level: toApiLogLevel(this.state.log_level),
- map_directory: this.state.map_directory,
+ map_destination: this.state.map_destination,
nav_color: this.state.navColor,
throttle_on_startup: this.state.throttle_on_startup,
- videos_directory: this.state.videos_directory,
- zims_directory: this.state.zims_directory,
+ videos_destination: this.state.videos_destination,
+ zims_destination: this.state.zims_destination,
}
try {
const response = await saveSettings(settings);
@@ -304,7 +304,7 @@ export class SettingsPage extends React.Component {
}
const {
- archive_directory,
+ archive_destination,
disabled,
download_on_startup,
download_timeout,
@@ -314,13 +314,13 @@ export class SettingsPage extends React.Component {
hotspot_ssid,
ignore_outdated_zims,
log_level,
- map_directory,
+ map_destination,
navColor,
pending,
qrCodeValue,
throttle_on_startup,
- videos_directory,
- zims_directory,
+ videos_destination,
+ zims_destination,
} = this.state;
const qrButton =
;
@@ -468,9 +468,9 @@ export class SettingsPage extends React.Component {
this.handleInputChange(e, 'archive_directory', d.value)}
+ onChange={(e, d) => this.handleInputChange(e, 'archive_destination', d.value)}
/>
@@ -479,9 +479,9 @@ export class SettingsPage extends React.Component {
this.handleInputChange(e, 'videos_directory', d.value)}
+ onChange={(e, d) => this.handleInputChange(e, 'videos_destination', d.value)}
/>
@@ -492,9 +492,9 @@ export class SettingsPage extends React.Component {
this.handleInputChange(e, 'map_directory', d.value)}
+ onChange={(e, d) => this.handleInputChange(e, 'map_destination', d.value)}
/>
@@ -503,9 +503,9 @@ export class SettingsPage extends React.Component {
this.handleInputChange(e, 'zims_directory', d.value)}
+ onChange={(e, d) => this.handleInputChange(e, 'zims_destination', d.value)}
/>
diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js
index c8368885..0f4c86ff 100644
--- a/app/src/hooks/customHooks.js
+++ b/app/src/hooks/customHooks.js
@@ -17,7 +17,7 @@ import {
getStatus,
getVideo,
getVideosStatistics,
- searchArchives,
+ searchArchives, searchChannels,
searchDirectories,
searchVideos,
searchZim,
@@ -1230,4 +1230,31 @@ export const useDockerized = () => {
export const useCalcQuery = () => {
const [calc, setCalc] = useOneQuery('calc');
return {calc, setCalc}
+}
+
+export const useSearchChannels = (defaultTagNames) => {
+ const [tagNames, setTagNames] = useState(defaultTagNames || []);
+ const [channels, setChannels] = useState([]);
+ const [loading, setLoading] = useState(false);
+
+ const localSearchChannels = async () => {
+ setLoading(true);
+ try {
+ const {channels: newChannels} = await searchChannels(tagNames);
+ setChannels(newChannels);
+ } finally {
+ setLoading(false);
+ }
+ };
+
+ useEffect(() => {
+ localSearchChannels();
+ }, [tagNames]);
+
+ return {
+ tagNames,
+ setTagNames,
+ channels,
+ loading,
+ }
}
\ No newline at end of file
diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py
index aa4a2bcd..29eeaa4b 100644
--- a/modules/archive/__init__.py
+++ b/modules/archive/__init__.py
@@ -49,17 +49,20 @@ class ArchiveDownloader(Downloader, ABC):
singlefile, readability, screenshot = await self.do_archive(download)
archive: Archive = await lib.model_archive_result(download.url, singlefile, readability, screenshot)
+ archive_id = archive.id
- if download.settings and (tag_names := download.settings.get('tag_names')):
- for name in tag_names:
- archive.add_tag(name)
+ with get_db_session() as session:
+ archive = Archive.find_by_id(archive_id, session)
+ if download.settings and (tag_names := download.settings.get('tag_names')):
+ for name in tag_names:
+ archive.add_tag(name, session)
- if session := Session.object_session(archive):
- session.commit()
+ if session := Session.object_session(archive):
+ session.commit()
- logger.info(f'Successfully downloaded Archive {download.url} {archive}')
+ logger.info(f'Successfully downloaded Archive {download.url} {archive}')
- return DownloadResult(success=True, location=f'/archive/{archive.id}')
+ return DownloadResult(success=True, location=f'/archive/{archive.id}')
@optional_session
def already_downloaded(self, *urls: List[str], session: Session = None) -> List:
diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py
index 4065cd8f..0898d8ee 100644
--- a/modules/archive/conftest.py
+++ b/modules/archive/conftest.py
@@ -82,7 +82,7 @@ def archive_factory(test_session, archive_directory, make_files_structure):
archive.validate()
for tag_name in tag_names:
- archive.add_tag(tag_name)
+ archive.add_tag(tag_name, test_session)
return archive
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 45992410..f67e6277 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -205,7 +205,8 @@ async def model_archive_result(url: str, singlefile: str, readability: dict, scr
archive.file_group.download_datetime = now()
archive.url = url
archive.domain = get_or_create_domain(session, url)
- session.flush()
+ archive.flush()
+ archive.domain.flush()
return archive
diff --git a/modules/archive/models.py b/modules/archive/models.py
index 05ea88a0..0e5a311f 100644
--- a/modules/archive/models.py
+++ b/modules/archive/models.py
@@ -315,8 +315,8 @@ class Archive(Base, ModelHelper):
archive = model_archive(file_group, session)
return archive
- def add_tag(self, tag_id_or_name: int | str) -> TagFile:
- return self.file_group.add_tag(tag_id_or_name)
+ def add_tag(self, tag_id_or_name: int | str, session: Session = None) -> TagFile:
+ return self.file_group.add_tag(tag_id_or_name, session)
class Domain(Base, ModelHelper):
diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py
index db0e4139..0af2e71e 100644
--- a/modules/archive/test/test_api.py
+++ b/modules/archive/test/test_api.py
@@ -84,7 +84,7 @@ def test_archives_search(test_session, archive_directory, archive_factory, test_
@pytest.mark.asyncio
async def test_search_archive_tags(test_session, test_async_client, archive_factory, tag_factory):
"""Tagged Archives can be searched."""
- tag = tag_factory()
+ tag = await tag_factory()
archive_factory(domain='example.com', tag_names=[tag.name, ])
test_session.commit()
diff --git a/modules/videos/api.py b/modules/videos/api.py
index a1ed068a..21a64860 100644
--- a/modules/videos/api.py
+++ b/modules/videos/api.py
@@ -2,12 +2,15 @@ from http import HTTPStatus
from sanic import Blueprint
from sanic.request import Request
+from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
from wrolpi.common import logger
from . import lib, schema
from .channel.api import channel_bp
+from .lib import format_videos_destination
+from .models import Channel
from .video.api import video_bp
content_bp = Blueprint('VideoContent', '/api/videos')
@@ -25,3 +28,20 @@ logger = logger.getChild(__name__)
async def statistics(_: Request):
ret = await lib.get_statistics()
return json_response(ret, HTTPStatus.OK)
+
+
+@content_bp.post('/tag_info')
+@openapi.definition(
+ description='Get data about tagging a Channel',
+ body=schema.ChannelTagInfoRequest,
+)
+@validate(schema.ChannelTagInfoRequest)
+async def channel_tag_info(_: Request, body: schema.ChannelTagInfoRequest):
+ from wrolpi.tags import Tag
+ channel = Channel.find_by_id(body.channel_id) if body.channel_id else None
+ channel_name = channel.name if channel else None
+ channel_url = channel.url if channel else None
+ tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None
+ videos_destination = format_videos_destination(channel_name, tag_name, channel_url)
+ ret = dict(videos_destination=videos_destination)
+ return json_response(ret)
diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py
index 4bc5ade3..72b6db22 100644
--- a/modules/videos/channel/api.py
+++ b/modules/videos/channel/api.py
@@ -17,7 +17,7 @@ from wrolpi.vars import PYTEST
from . import lib
from .. import schema
from ..errors import UnknownChannel
-from ..lib import save_channels_config, format_videos_destination
+from ..lib import save_channels_config
from ..models import Channel
channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels')
@@ -78,9 +78,9 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
@validate(schema.ChannelPutRequest)
@run_after(save_channels_config)
@wrol_mode_check
-def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
+async def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
body.directory = pathlib.Path(body.directory) if body.directory else None
- channel = lib.update_channel(data=body, channel_id=channel_id)
+ channel = await lib.update_channel(data=body, channel_id=channel_id)
return response.raw('', HTTPStatus.NO_CONTENT,
headers={'Location': f'/api/videos/channels/{channel.id}'})
@@ -138,32 +138,14 @@ async def channel_download(_: Request, channel_id: int, body: schema.ChannelDown
return response.empty()
-@channel_bp.post('/
/tag_info')
-@openapi.definition(
- description='Get Channel Tag Info',
- body=schema.ChannelTagInfoRequest,
-)
-@validate(schema.ChannelTagInfoRequest)
-@run_after(save_channels_config)
-@wrol_mode_check
-async def channel_tag_info(_: Request, channel_id: int, body: schema.ChannelTagInfoRequest):
- from wrolpi.tags import Tag
- channel_name = Channel.find_by_id(channel_id).name
- tag_name = Tag.find_by_name(body.tag_name).name if body.tag_name else None
- videos_destination = format_videos_destination(channel_name, tag_name)
- ret = dict(videos_destination=videos_destination)
- return json_response(ret)
-
-
@channel_bp.post('//tag')
@openapi.definition(
- description='Tag Channel',
+ description='Tag/untag a Channel with a single tag',
body=schema.ChannelTagRequest,
)
@openapi.response(HTTPStatus.NO_CONTENT)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@validate(schema.ChannelTagRequest)
-@run_after(save_channels_config)
@wrol_mode_check
async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagRequest):
directory = None
@@ -174,3 +156,17 @@ async def channel_tag(_: Request, channel_id: int, body: schema.ChannelTagReques
await lib.tag_channel(body.tag_name, directory, channel_id)
return response.raw('', HTTPStatus.NO_CONTENT)
+
+
+@channel_bp.post('/search')
+@openapi.definition(
+ description='Search Channels',
+ body=schema.ChannelSearchRequest,
+)
+@validate(schema.ChannelSearchRequest)
+async def channel_search(_: Request, body: schema.ChannelSearchRequest):
+ channels = await lib.search_channels(body.tag_names)
+ ret = dict(
+ channels=channels,
+ )
+ return json_response(ret)
\ No newline at end of file
diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py
index ed7ee5ee..fdca41fb 100644
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@@ -7,10 +7,11 @@ from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.common import run_after, logger, \
- get_media_directory, wrol_mode_check
+ get_media_directory, wrol_mode_check, background_task
from wrolpi.db import get_db_curs, optional_session, get_db_session
from wrolpi.downloader import save_downloads_config, download_manager
-from wrolpi.errors import UnknownDirectory, APIError, ValidationError, RefreshConflict
+from wrolpi.errors import APIError, ValidationError, RefreshConflict
+from wrolpi.vars import PYTEST
from .. import schema
from ..common import check_for_channel_conflicts
from ..errors import UnknownChannel, ChannelDirectoryConflict
@@ -115,22 +116,10 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No
@run_after(save_channels_config)
@optional_session
-def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
+async def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
"""Update a Channel's DB record"""
channel = Channel.find_by_id(channel_id, session)
- # Only update directory if it was empty
- if data.directory and not channel.directory:
- data.directory = get_media_directory() / data.directory
- if not data.directory.is_dir():
- if data.mkdir:
- data.directory.mkdir()
- else:
- raise UnknownDirectory()
- elif data.directory:
- # Keep the old directory because the user can't change a Channel's directory.
- data.directory = channel.directory
-
# Verify that the URL/Name/directory aren't taken
check_for_channel_conflicts(
session,
@@ -140,11 +129,27 @@ def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_
directory=data.directory,
)
+ data = data.__dict__
+ old_directory = pathlib.Path(channel.directory)
+ new_directory = get_media_directory() / data.pop('directory')
+
# Apply the changes now that we've OK'd them
- channel.update(data.__dict__)
+ channel.update(data)
session.commit()
+ async def _():
+ await channel.move_channel(new_directory, session, send_events=True)
+
+ if new_directory != old_directory:
+ if not new_directory.is_dir():
+ new_directory.mkdir(parents=True)
+
+ if PYTEST:
+ await _()
+ else:
+ background_task(_())
+
return channel
@@ -252,7 +257,7 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f
@wrol_mode_check
@optional_session
-async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id: int, session: Session = None):
+async def tag_channel(tag_name: str | None, directory: pathlib.Path | None, channel_id: int, session: Session = None):
"""Add a Tag to a Channel, or remove a Tag from a Channel if no `tag_name` is provided.
Move the Channel to the new directory, if provided."""
@@ -261,29 +266,39 @@ async def tag_channel(tag_name: str, directory: pathlib.Path | None, channel_id:
if directory and flags.refreshing.is_set():
raise RefreshConflict('Refusing to move channel while file refresh is in progress')
- channel = Channel.find_by_id(channel_id, session=session)
+ channel = Channel.find_by_id(channel_id, session)
if tag_name:
- tag = Tag.find_by_name(tag_name, session=session)
+ tag = Tag.find_by_name(tag_name, session)
channel.tag_id = tag.id
channel.tag = tag
else:
channel.tag = channel.tag_id = None
- session.flush([channel, ])
+ channel.flush()
+ session.commit()
# Do not move Channel files into a directory with files.
- if directory and directory.is_dir() and next(directory.iterdir(), None):
+ if directory and directory != channel.directory and directory.is_dir() and next(directory.iterdir(), None):
raise ChannelDirectoryConflict('Channel directory already exists and is not empty')
# Only move Channel when requested.
if directory:
# Move to newly defined directory only if necessary.
- logger.info(f'Tagging {channel} with {tag_name} moving to {directory} from {channel.directory}')
directory.mkdir(parents=True, exist_ok=True)
if channel.directory != directory:
- await channel.move(directory, session)
+ coro = channel.move_channel(directory, session, send_events=True)
+ if PYTEST:
+ await coro
+ else:
+ background_task(coro)
else:
logger.info(f'Tagging {channel} with {tag_name}')
- session.commit()
+
+@optional_session
+async def search_channels(tag_names: List[str], session: Session) -> List[Channel]:
+ """Search Tagged Channels."""
+ from wrolpi.tags import Tag
+ channels = session.query(Channel).join(Tag).filter(Tag.name.in_(tag_names)).all()
+ return channels
diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py
index c615bb4f..1309d776 100644
--- a/modules/videos/channel/test/test_api.py
+++ b/modules/videos/channel/test/test_api.py
@@ -1,6 +1,7 @@
import asyncio
import json
import pathlib
+import shutil
import tempfile
from datetime import timedelta
from http import HTTPStatus
@@ -96,7 +97,8 @@ def test_video_file_name(test_session, simple_video, test_client):
assert resp.json['file_group']['video'].get('stem') == 'simple_video'
-def test_channel_conflicts(test_client, test_session, test_directory):
+@pytest.mark.asyncio
+async def test_channel_conflicts(test_async_client, test_session, test_directory):
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
new_channel = dict(
@@ -105,11 +107,11 @@ def test_channel_conflicts(test_client, test_session, test_directory):
url='https://example.com/channel1',
)
- def _post_channel(channel):
- return test_client.post('/api/videos/channels', content=json.dumps(channel))
+ async def _post_channel(channel):
+ return await test_async_client.post('/api/videos/channels', content=json.dumps(channel))
# Create it
- request, response = _post_channel(new_channel)
+ request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.CREATED
# Name is an exact match
@@ -119,7 +121,7 @@ def test_channel_conflicts(test_client, test_session, test_directory):
directory=channel_directory2,
name='Example Channel 1',
)
- request, response = _post_channel(new_channel)
+ request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'cause': {'code': 'CHANNEL_NAME_CONFLICT',
'error': 'The channel name is already taken.',
@@ -133,7 +135,7 @@ def test_channel_conflicts(test_client, test_session, test_directory):
directory=channel_directory,
name='name is fine',
)
- request, response = _post_channel(new_channel)
+ request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'cause': {'code': 'CHANNEL_DIRECTORY_CONFLICT',
'error': 'The directory is already used by another channel.',
@@ -150,7 +152,7 @@ def test_channel_conflicts(test_client, test_session, test_directory):
name='name is fine',
url='https://example.com/channel1',
)
- request, response = _post_channel(new_channel)
+ request, response = await _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'cause': {'code': 'CHANNEL_URL_CONFLICT',
'error': 'The URL is already used by another channel.',
@@ -384,7 +386,8 @@ def test_search_videos_channel(test_client, test_session, video_factory):
dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
-def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
+@pytest.mark.asyncio
+async def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
for i in range(50):
video_factory(channel_id=simple_channel.id)
@@ -406,8 +409,9 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
]
last_ids = []
for offset, video_count in tests:
- _, response = assert_video_search(channel_id=simple_channel.id, order_by='published_datetime', offset=offset,
- limit=20)
+ _, response = await assert_video_search(channel_id=simple_channel.id, order_by='published_datetime',
+ offset=offset,
+ limit=20)
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
current_ids = [i['id'] for i in response.json['file_groups']]
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
@@ -416,7 +420,7 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
@pytest.mark.asyncio
async def test_create_channel_download_api(test_async_client, test_session, simple_channel, tag_factory):
- tag = tag_factory()
+ tag = await tag_factory()
# Create Download.
body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
@@ -465,7 +469,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
videos_directory = get_videos_directory()
channel = channel_factory(name='Channel Name', download_frequency=120)
channel_directory = channel.directory
- tag = tag_factory('Tag Name')
+ tag = await tag_factory('Tag Name')
v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id)
# Create recurring download which uses the Channel's directory.
download = download_manager.recurring_download('https://example.com/1', 60, test_downloader.name,
@@ -532,7 +536,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
- # Channel can be Tagged, but not move directories.
+ # Channel can be Tagged, without moving directories.
body = dict(tag_name=tag.name)
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
@@ -545,3 +549,79 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
d1, d2 = test_session.query(Download).all()
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
+
+
+@pytest.mark.asyncio
+async def test_move_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory,
+ video_factory, test_channels_config, test_download_manager, test_downloader):
+ """A Channel can be moved."""
+ channel = channel_factory(name='Channel Name')
+ vid1 = video_factory(channel_id=channel.id)
+ vid2 = video_factory(channel_id=channel.id)
+ test_session.commit()
+
+ assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
+ assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/'))
+ assert str(vid2.video_path).startswith(str(test_directory / 'videos/Channel Name/'))
+
+ body = dict(
+ name=channel.name,
+ directory=str(test_directory / 'videos/New Directory'),
+ )
+ request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
+ channel = Channel.find_by_id(channel.id)
+ assert str(channel.directory) == str(test_directory / 'videos/New Directory')
+ assert str(vid1.video_path).startswith(str(test_directory / 'videos/New Directory/'))
+ assert str(vid2.video_path).startswith(str(test_directory / 'videos/New Directory/'))
+
+
+@pytest.mark.asyncio
+async def test_move_channel_after_terminal_move(test_async_client, test_session, test_directory, channel_factory,
+ tag_factory, video_factory, test_channels_config, test_download_manager,
+ test_downloader):
+ """A Channel can be moved in the UI after being moved in the Terminal."""
+ channel = channel_factory(name='Channel Name')
+ vid1 = video_factory(channel_id=channel.id)
+ test_session.commit()
+
+ assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
+ assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/'))
+
+ # mv 'videos/Channel Name' 'videos/New Directory'
+ shutil.move(channel.directory, test_directory / 'videos/New Directory')
+
+ body = dict(
+ name=channel.name,
+ directory=str(test_directory / 'videos/New Directory'),
+ )
+ request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
+ assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
+ channel = Channel.find_by_id(channel.id)
+ vid1 = test_session.query(Video).one()
+ assert str(channel.directory) == str(test_directory / 'videos/New Directory')
+ assert str(vid1.video_path).startswith(str(test_directory / 'videos/New Directory/'))
+
+
+@pytest.mark.asyncio
+async def test_search_tagged_channels(test_async_client, test_session, channel_factory, tag_factory,
+ video_factory):
+ """Tagged Channels can be searched."""
+ tag = await tag_factory()
+ test_session.commit()
+
+ # Can search when empty.
+ body = dict(tag_names=[tag.name])
+ request, response = await test_async_client.post('/api/videos/channels/search', json=body)
+ assert response.status_code == HTTPStatus.OK
+ assert len(response.json['channels']) == 0, 'No Channels should be tagged'
+
+ channel1 = channel_factory(tag_name=tag.name)
+ channel2 = channel_factory()
+ test_session.commit()
+
+ body = dict(tag_names=[tag.name])
+ request, response = await test_async_client.post('/api/videos/channels/search', json=body)
+ assert response.status_code == HTTPStatus.OK
+ assert len(response.json['channels']) == 1, 'Only one Channel is tagged'
+ assert response.json['channels'][0]['id'] == channel1.id
diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py
index c9100110..3a36d021 100644
--- a/modules/videos/channel/test/test_lib.py
+++ b/modules/videos/channel/test/test_lib.py
@@ -1,6 +1,7 @@
from typing import List
import pytest
+import yaml
from modules.videos import schema
from modules.videos.channel import lib
@@ -124,3 +125,23 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact
await assert_search('foo', ['foo', 'foo bar', 'foobar'])
await assert_search('bar', ['foobar', 'foo bar', 'bar'], True)
+
+
+@pytest.mark.asyncio
+async def test_tag_channel_existing(test_async_client, test_session, test_directory, channel_factory, tag_factory,
+ video_factory, test_channels_config):
+ """A Channel can already exist in a Tag's directory and still be tagged."""
+ channel_directory = test_directory / 'videos/one/Channel Name'
+ channel = channel_factory(name='Channel Name', download_frequency=120, directory=channel_directory)
+ video_factory(channel_id=channel.id)
+ test_session.commit()
+ assert str(channel.directory).startswith(f'{test_directory}/videos/one/')
+
+ tag = await tag_factory()
+ await lib.tag_channel(tag.name, channel_directory, channel.id, test_session)
+
+ # Channel's tag is saved to the config.
+ save_channels_config(test_session)
+ with test_channels_config.open() as fh:
+ config = yaml.load(fh, Loader=yaml.Loader)
+ assert config['channels'][0]['tag_name'] == tag.name
diff --git a/modules/videos/common.py b/modules/videos/common.py
index 8a4307ef..9ba724f2 100644
--- a/modules/videos/common.py
+++ b/modules/videos/common.py
@@ -17,7 +17,7 @@ from sqlalchemy.orm import Session
from wrolpi.captions import FFMPEG_BIN
from wrolpi.cmd import FFPROBE_BIN
-from wrolpi.common import logger, get_media_directory, get_wrolpi_config
+from wrolpi.common import logger, get_media_directory
from wrolpi.dates import seconds_to_timestamp
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
@@ -204,7 +204,7 @@ def is_valid_poster(poster_path: Path) -> bool:
async def ffprobe_json(video_path: Union[Path, str]) -> dict:
"""Extract video file metadata using ffprobe.
- >>> ffprobe_json('video')
+ >>> ffprobe_json('video.mp4')
{
'chapters': [],
'format': {},
@@ -270,39 +270,6 @@ def get_video_duration(video_path: Path) -> Optional[int]:
return duration
-def check_for_video_corruption(video_path: Path) -> bool:
- """Uses ffprobe to check for specific ways a video file can be corrupt.
-
- Also attempts to screenshot the video near the end of it's duration, this check if the video is corrupted somewhere
- in the middle."""
- video_path = Path(video_path) if not isinstance(video_path, Path) else video_path
-
- if not video_path.is_file():
- raise FileNotFoundError(f'{video_path} does not exist!')
- if not FFPROBE_BIN:
- raise SystemError('ffprobe is not installed!')
-
- # Just read the video using FFMPEG, this prints out useful information about the video.
- cmd = (FFPROBE_BIN, str(video_path))
- try:
- proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- except subprocess.CalledProcessError as e:
- logger.warning(f'FFPROBE failed to check for corruption with stderr: {e.stderr.decode()}')
- return True # video is corrupt.
-
- messages = (
- b'Invalid NAL unit size',
- b'Error splitting the input into NAL units',
- b'Invalid data found when processing input',
- )
- corrupt = False
- for error in messages:
- if error in proc.stderr:
- logger.warning(f'Possible video corruption ({error.decode()}): {video_path}')
- corrupt = True
- return corrupt
-
-
async def update_view_counts_and_censored(channel_id: int):
"""Update view_count for all Videos in a channel using its info_json file. Also sets FileGroup.censored
if Video is no longer available on the Channel."""
diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py
index 0e2f7240..6a3e71b1 100644
--- a/modules/videos/conftest.py
+++ b/modules/videos/conftest.py
@@ -40,14 +40,16 @@ def channel_factory(test_session, test_directory):
def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None,
directory: pathlib.Path = None, tag_name: str = None):
name = name or str(uuid4())
- tag = Tag.get_by_name(tag_name) if tag_name else None
+ tag = Tag.find_by_name(tag_name) if tag_name else None
channel = Channel(
name=name,
url=url or f'https://example.com/{name}',
source_id=source_id,
tag=tag,
+ tag_id=tag.id if tag else None,
)
- channel.directory = directory or format_videos_destination(name, tag, channel.url)
+ tag_name = tag.name if tag else None
+ channel.directory = directory or format_videos_destination(name, tag_name, channel.url)
channel.directory.mkdir(exist_ok=True, parents=True)
test_session.add(channel)
test_session.flush([channel])
@@ -242,12 +244,12 @@ def video_with_search_factory(test_session, test_directory, video_file_factory):
@pytest.fixture
-def assert_video_search(test_client):
+def assert_video_search(test_async_client):
"""A fixture which performs a video search request against the API.
If assert_* params are passed, the response is checked."""
- def _search_videos(
+ async def _search_videos(
assert_total: int = None,
assert_ids: List[int] = None,
assert_paths: List[str] = None,
@@ -272,7 +274,7 @@ def assert_video_search(test_client):
if channel_id is not None:
content['channel_id'] = channel_id
- request, response = test_client.post('/api/videos/search', content=json.dumps(content))
+ request, response = await test_async_client.post('/api/videos/search', content=json.dumps(content))
assert response.json
assert response.status_code == HTTPStatus.OK
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index 2979846f..c77b0eab 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -4,7 +4,7 @@ import os.path
import pathlib
import traceback
from abc import ABC
-from typing import Tuple, List, Dict
+from typing import Tuple, List, Dict, Optional
import yt_dlp.utils
from sqlalchemy.orm import Session
@@ -64,6 +64,7 @@ def extract_info(url: str, ydl: YoutubeDL = YDL, process=False) -> dict:
def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
"""Get filename from YoutubeDL. Separated for testing."""
dir_name, file_name = os.path.split(ydl.prepare_filename(entry))
+ file_name = escape_file_name(file_name)
file_name = trim_file_name(file_name)
return f'{dir_name}/{file_name}'
@@ -241,8 +242,8 @@ class VideoDownloader(Downloader, ABC):
# Video may have been downloaded previously, get its location for error reporting.
with get_db_session() as session:
- videos = session.query(Video).join(FileGroup).filter_by(url=url).all()
- location = videos[0].location if videos else None
+ video_ = Video.get_by_url(url, session)
+ location = video_.location if video_ else None
try:
download.info_json = download.info_json or extract_info(url)
@@ -257,62 +258,7 @@ class VideoDownloader(Downloader, ABC):
if not download.info_json:
raise ValueError(f'Cannot download video with no info_json.')
- found_channel = None
-
- # Look for Channel using channel data first, fallback to uploader data.
- channel_name = download.info_json.get('channel') or download.info_json.get('uploader')
- channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id')
- channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url')
- channel = None
- if channel_name or channel_source_id or channel_url:
- # Try to find the channel via info_json from yt-dlp.
- try:
- channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name)
- if channel:
- found_channel = 'yt_dlp'
- logger.debug(f'Found a channel with source_id {channel=}')
- except UnknownChannel:
- # Can't find a channel, use the no channel directory.
- pass
-
- settings = download.settings or dict()
- destination = settings.get('destination')
- if not channel and destination:
- # Destination may find Channel if not already found.
- try:
- channel = get_channel(directory=destination, return_dict=False)
- found_channel = 'download_settings_directory'
- logger.debug(f'Found a channel that shares the destination directory {channel=}')
- except UnknownChannel:
- # Destination must not be a channel.
- pass
-
- local_channel_id = settings.get('channel_id')
- channel_url = settings.get('channel_url')
- if not channel and (local_channel_id or channel_url):
- # Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
- logger.info(f'Using download.settings to find channel')
- try:
- channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False)
- logger.debug(f'Found channel with channel url {channel=}')
- except UnknownChannel:
- # We do not need a Channel if we have a destination directory.
- if not destination:
- raise
- if channel:
- found_channel = 'download_settings'
-
- channel_id = channel.id if channel else None
- channel_directory = channel.directory if channel else None
-
- if found_channel == 'yt_dlp':
- logger.debug(f'Found {channel} using yt_dlp')
- elif found_channel == 'download_settings':
- logger.info(f'Found {channel} using Download.settings')
- elif found_channel == 'download_settings_directory':
- logger.info(f'Found {channel} using Download.settings directory')
- else:
- logger.warning('Could not find channel')
+ channel, channel_directory, channel_id, destination, settings = await self._get_channel(download)
if destination:
# Download to the directory specified in the settings.
@@ -480,6 +426,63 @@ class VideoDownloader(Downloader, ABC):
)
return result
+ @staticmethod
+ async def _get_channel(download: Download) \
+ -> Tuple[Optional[Channel], Optional[pathlib.Path], Optional[int], Optional[pathlib.Path], dict]:
+ found_channel = None
+ # Look for Channel using channel data first, fallback to uploader data.
+ channel_name = download.info_json.get('channel') or download.info_json.get('uploader')
+ channel_source_id = download.info_json.get('channel_id') or download.info_json.get('uploader_id')
+ channel_url = download.info_json.get('channel_url') or download.info_json.get('uploader_url')
+ channel = None
+ if channel_name or channel_source_id or channel_url:
+ # Try to find the channel via info_json from yt-dlp.
+ try:
+ channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name)
+ if channel:
+ found_channel = 'yt_dlp'
+ logger.debug(f'Found a channel with source_id {channel=}')
+ except UnknownChannel:
+ # Can't find a channel, use the no channel directory.
+ pass
+ settings = download.settings or dict()
+ destination = settings.get('destination')
+ if not channel and destination:
+ # Destination may find Channel if not already found.
+ try:
+ channel = get_channel(directory=destination, return_dict=False)
+ found_channel = 'download_settings_directory'
+ logger.debug(f'Found a channel that shares the destination directory {channel=}')
+ except UnknownChannel:
+ # Destination must not be a channel.
+ pass
+ channel_id = settings.get('channel_id')
+ channel_url = settings.get('channel_url')
+ if not channel and (channel_id or channel_url):
+ # Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
+ logger.info(f'Using download.settings to find channel')
+ try:
+ channel = get_channel(channel_id=channel_id, url=channel_url, return_dict=False)
+ logger.debug(f'Found channel with channel url {channel=}')
+ except UnknownChannel:
+ # We do not need a Channel if we have a destination directory.
+ if not destination:
+ raise
+ if channel:
+ found_channel = 'download_settings'
+ channel_id = channel.id if channel else None
+ channel_directory = channel.directory if channel else None
+ if found_channel == 'yt_dlp':
+ logger.debug(f'Found {channel} using yt_dlp')
+ elif found_channel == 'download_settings':
+ logger.info(f'Found {channel} using Download.settings')
+ elif found_channel == 'download_settings_directory':
+ logger.info(f'Found {channel} using Download.settings directory')
+ else:
+ logger.warning('Could not find channel')
+
+ return channel, channel_directory, channel_id, destination, settings
+
@staticmethod
def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
"""Get the full path of a video file from its URL."""
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 265031d5..f79f15c8 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -3,6 +3,7 @@ import dataclasses
import html
import pathlib
import re
+import traceback
from typing import Generator
from typing import Tuple, Optional
@@ -15,7 +16,7 @@ from modules.videos.models import Video
from wrolpi import before_startup, dates, flags
from wrolpi.captions import extract_captions
from wrolpi.common import ConfigFile, limit_concurrent, get_wrolpi_config, extract_domain, \
- escape_file_name, get_media_directory
+ escape_file_name, get_media_directory, background_task
from wrolpi.dates import Seconds, from_timestamp
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download
@@ -379,6 +380,22 @@ def save_channels_config(session: Session = None):
channel_import_logger = logger.getChild('channel_import')
+async def fetch_channel_source_id(channel_id: int):
+ # If we can download from a channel, we must have its source_id.
+ try:
+ with get_db_session() as session:
+ from modules.videos.channel.lib import get_channel
+ channel = Channel.find_by_id(channel_id)
+ channel.source_id = channel.source_id or get_channel_source_id(channel.url)
+ if not channel.source_id:
+ channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
+ else:
+ session.commit()
+ save_channels_config(session)
+ except Exception as e:
+ logger.error(f'Failed to get Channel source id of id={channel_id}', exc_info=e)
+
+
@before_startup
@limit_concurrent(1)
def import_channels_config():
@@ -387,13 +404,17 @@ def import_channels_config():
logger.warning('Skipping import_channels_config for this test')
return
+ from .channel.lib import get_channel
channel_import_logger.info('Importing channels config')
try:
config = get_channels_config()
channels = config.channels
channel_directories = [i['directory'] for i in channels]
+ if len(channel_directories) != len(set(channel_directories)):
+ logger.error('Refusing to import channels config because it contains duplicate directories!')
+ return
+ updated_channel_ids = set()
- save_config = False
with get_db_session(commit=True) as session:
for data in channels:
if isinstance(data, str):
@@ -405,45 +426,37 @@ def import_channels_config():
for option in (i for i in REQUIRED_OPTIONS if i not in data):
raise ConfigError(f'Channel "{directory}" is required to have "{option}"')
- channel = session.query(Channel).filter(
- or_(
- Channel.directory == str(directory),
- Channel.directory == str(data['directory']),
- Channel.directory == str(directory.relative_to(get_media_directory())),
- ),
- ).one_or_none()
+ # Try to find Channel by directory because it is unique.
+ channel = Channel.get_by_path(directory, session)
if not channel:
- # Channel not yet in the DB, add it.
- channel = Channel(directory=directory)
- session.add(channel)
- session.flush([channel, ])
- # TODO refresh the files in the channel.
- logger.warning(f'Creating new Channel from config {channel}')
-
- # Only name and directory are required
- channel.name = data['name']
- channel.directory = directory
-
- # A URL should not be an empty string
- data['url'] = data['url'] or None
+ try:
+ # Try to find Channel using other attributes before creating new Channel.
+ channel = get_channel(
+ session,
+ source_id=data.get('source_id'),
+ url=data.get('url'),
+ directory=str(directory),
+ return_dict=False,
+ )
+ except UnknownChannel:
+ # Channel not yet in the DB, add it.
+ channel = Channel(directory=directory)
+ session.add(channel)
+ channel.flush()
+ # TODO refresh the files in the channel.
+ logger.warning(f'Creating new Channel from config {channel}')
# Copy existing channel data, update all values from the config. This is necessary to clear out
# values not in the config.
full_data = channel.dict()
full_data.update(data)
channel.update(full_data)
+ updated_channel_ids.add(channel.id)
if not channel.source_id and channel.url and flags.have_internet.is_set():
# If we can download from a channel, we must have its source_id.
- try:
- channel.source_id = get_channel_source_id(channel.url)
- save_config = True
- if not channel.source_id:
- channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
- else:
- session.commit()
- except Exception as e:
- logger.error(f'Failed to get Channel source id of {channel}', exc_info=e)
+ logger.info(f'Fetching channel source id for {channel}')
+ background_task(fetch_channel_source_id(channel.id))
channel_import_logger.debug(f'Updated {repr(channel.name)}'
f' url={channel.url}'
@@ -454,17 +467,13 @@ def import_channels_config():
with get_db_session(commit=True) as session:
# Delete any Channels that were deleted from the config.
for channel in session.query(Channel):
- if str(channel.directory) not in channel_directories:
+ if channel.id not in updated_channel_ids:
logger.warning(f'Deleting {channel} because it is not in the config.')
channel.delete_with_videos()
# Create any missing Downloads. Associated Downloads with any necessary Channels.
link_channel_and_downloads()
- if save_config:
- # Information about the channel was fetched, store it.
- save_channels_config()
-
except Exception as e:
channel_import_logger.warning('Failed to load channels config!', exc_info=e)
if PYTEST:
@@ -544,7 +553,8 @@ async def get_statistics():
COUNT(v.id) FILTER ( WHERE v.have_comments = TRUE ) AS "have_comments",
COUNT(v.id) FILTER ( WHERE v.have_comments = FALSE AND v.comments_failed = FALSE
and fg.censored = false and fg.url is not null) AS "missing_comments",
- COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments"
+ COUNT(v.id) FILTER ( WHERE v.comments_failed = TRUE ) AS "failed_comments",
+ COUNT(v.id) FILTER ( WHERE fg.censored = TRUE ) as "censored_videos"
FROM
video v
LEFT JOIN file_group fg on v.file_group_id = fg.id
@@ -583,9 +593,11 @@ async def get_statistics():
curs.execute('''
SELECT
- COUNT(id) AS "channels"
+ COUNT(c.id) AS "channels",
+ COUNT(c.id) FILTER ( WHERE c.tag_id IS NOT NULL ) AS "tagged_channels"
FROM
- channel
+ channel c
+ LEFT JOIN public.tag t on t.id = c.tag_id
''')
channel_stats = dict(curs.fetchone())
ret = dict(statistics=dict(
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 1b9b598e..0442fe14 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -1,4 +1,5 @@
import json
+import json
import pathlib
from typing import Optional, Dict, List
@@ -7,11 +8,14 @@ from sqlalchemy.orm import relationship, Session, deferred
from sqlalchemy.orm.collections import InstrumentedList
from modules.videos.errors import UnknownVideo, UnknownChannel
+from wrolpi import flags
from wrolpi.captions import read_captions
-from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file
+from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file, \
+ unique_by_predicate
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download, download_manager, save_downloads_config
from wrolpi.errors import InvalidDownload
+from wrolpi.events import Events
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix, move
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
@@ -223,6 +227,7 @@ class Video(ModelHelper, Base):
# Can't validate if there is no video file.
logger.error(f'Unable to validate video {self.id} without primary file!')
+ logger.debug(f'Validating {self}')
from .lib import validate_video
try:
validate_video(self, self.channel.generate_posters if self.channel else False)
@@ -237,12 +242,14 @@ class Video(ModelHelper, Base):
# self.file_group.d_text is handled in `validate_video`.
if (comments := self.get_comments()) and len(comments) >= 5:
+ logger.debug(f'{self} has comments')
self.have_comments = True
# If this Video is in a Channel's directory, then it is part of that Channel.
session: Session = Session.object_session(self)
if session and (channel := Channel.get_by_path(self.video_path.parent, session)):
self.channel = channel
+ logger.debug(f'{self} has Channel {channel}')
@staticmethod
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
@@ -533,7 +540,7 @@ class Channel(ModelHelper, Base):
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id')
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'))
- tag = relationship('Tag')
+ tag = relationship('Tag', primaryjoin='Channel.tag_id==Tag.id')
def __repr__(self):
return f''
@@ -565,12 +572,9 @@ class Channel(ModelHelper, Base):
channel_id = self.id
data = data.copy()
- # Remove unused values.
- data.pop('move_to_tag_directory', None)
-
# URL should not be empty string.
- if 'url' in data:
- data['url'] = data['url'] or None
+ url = data.pop('url')
+ self.url = url or None
session: Session = Session.object_session(self)
@@ -664,23 +668,19 @@ class Channel(ModelHelper, Base):
with get_db_curs() as curs:
stmt = '''
SELECT
- SUM(size),
- MAX(size),
- COUNT(video.id),
- SUM(fg.length)
+ SUM(size) AS "size",
+ MAX(size) AS "largest_video",
+ COUNT(video.id) AS "video_count",
+ SUM(fg.length) AS "length",
+ -- Videos may use multiple tags.
+ COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
FROM video
LEFT JOIN file_group fg on fg.id = video.file_group_id
+ LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
WHERE channel_id = %(id)s
'''
curs.execute(stmt, dict(id=self.id))
- size, largest_video, video_count, length = curs.fetchone()
- statistics = dict(
- video_count=video_count,
- size=size,
- largest_video=largest_video,
- length=length,
- )
- return statistics
+ return dict(curs.fetchone())
async def refresh_files(self, send_events: bool = True):
"""Refresh all files within this Channel's directory. Mark this channel as refreshed."""
@@ -772,39 +772,72 @@ class Channel(ModelHelper, Base):
channel = session.query(Channel).filter_by(url=url).one_or_none()
return channel
- async def move(self, directory: pathlib.Path, session: Session):
+ def format_directory(self, tag_name: str) -> pathlib.Path:
+ from modules.videos.lib import format_videos_destination
+ return format_videos_destination(self.name, tag_name, self.url)
+
+ async def move_channel(self, directory: pathlib.Path, session: Session, send_events: bool = False):
"""Move the files of this Channel into a new directory."""
if not directory.is_dir():
raise FileNotFoundError(f'Destination directory does not exist: {directory}')
- if self.directory and not next(self.directory.iterdir(), None):
- raise FileExistsError(f'Refusing to move Channel when it has no files.')
old_directory = self.directory
self.directory = directory
- session.flush([self, ])
- # Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's
- # directory.
- downloads = list(self.downloads)
- downloads.extend(list(
- session.query(Download).filter(Download.settings.contains({'destination': str(old_directory)}))
- ))
- for download in downloads:
- # Get new copy of `settings` to avoid bug where it wouldn't update.
- settings = download.settings.copy() if download.settings else dict()
- if settings.get('destination'):
- download.settings = {**settings, 'destination': str(directory)}
+ def change_download_destinations(from_directory: pathlib.Path, to_directory: pathlib.Path):
+ downloads = list(self.downloads)
+ downloads.extend(Download.get_all_by_destination(from_directory))
+ downloads = unique_by_predicate(downloads, lambda i: i.id)
+ for download in downloads:
+ # Get new copy of `settings` to avoid bug where it wouldn't update.
+ settings = download.settings.copy() if download.settings else dict()
+ if settings.get('destination'):
+ logger.debug(f'Updating destination of {download}')
+ download.settings = {**settings, 'destination': str(to_directory)}
+ session.flush(downloads)
- session.commit()
+ # Only one tag can be moved at a time.
+ with flags.refreshing:
+ # Change destination of all Downloads of this Channel, or any Downloads which download into this Channel's
+ # directory.
+ change_download_destinations(old_directory, directory)
- # Save configs before move, this is because move imports configs.
- from modules.videos.lib import save_channels_config
- await save_downloads_config(session)
- save_channels_config(session)
- save_tags_config(session)
+ session.commit()
- # Move the contents of the Channel directory into the destination directory.
- await move(directory, *list(old_directory.iterdir()))
+ # Save configs before move, this is because move imports configs.
+ from modules.videos.lib import save_channels_config
+ await save_downloads_config(session)
+ save_channels_config(session)
+ save_tags_config(session)
- if old_directory.exists() and not next(old_directory.iterdir(), None):
- old_directory.rmdir()
+ # Move the contents of the Channel directory into the destination directory.
+ logger.info(f'Moving {self} from {repr(str(old_directory))}')
+
+ try:
+ if not old_directory.exists():
+ # Old directory does not exist, maintainer must have moved the Channel manually.
+ if PYTEST:
+ await refresh_files([old_directory, directory])
+ else:
+ background_task(refresh_files([old_directory, directory]))
+ if send_events:
+ Events.send_file_move_completed(f'Channel {repr(self.name)} was moved, but Tags were lost')
+ else:
+ # `move` also uses `flags.refreshing`
+ await move(directory, *list(old_directory.iterdir()))
+ if send_events:
+ Events.send_file_move_completed(f'Channel {repr(self.name)} was moved')
+ except Exception as e:
+ logger.error(f'Channel move failed! Reverting changes...', exc_info=e)
+ # Downloads must be moved back to the old directory.
+ change_download_destinations(directory, old_directory)
+ self.directory = old_directory
+ self.flush(session)
+ if send_events:
+ Events.send_file_move_failed(f'Moving Channel {self.name} has failed')
+ raise
+ finally:
+ session.commit()
+ if old_directory.exists() and not next(iter(old_directory.iterdir()), None):
+ # Old directory is empty, delete it.
+ old_directory.rmdir()
diff --git a/modules/videos/schema.py b/modules/videos/schema.py
index 4760149d..a21b1eee 100644
--- a/modules/videos/schema.py
+++ b/modules/videos/schema.py
@@ -49,6 +49,7 @@ class ChannelTagRequest:
@dataclass
class ChannelTagInfoRequest:
+ channel_id: Optional[int] = None
tag_name: Optional[str] = None
@@ -205,6 +206,7 @@ class VideoStatistics:
@dataclass
class ChannelStatistics:
channels: int
+ tagged_channels: int
@dataclass
@@ -222,3 +224,8 @@ class CensoredVideoRequest:
@dataclass
class CensoredVideoResponse:
videos: List[VideoWithChannel]
+
+
+@dataclass
+class ChannelSearchRequest:
+ tag_names: List[str] = field(default_factory=list)
diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py
index fd82274d..105fec06 100644
--- a/modules/videos/test/test_api.py
+++ b/modules/videos/test/test_api.py
@@ -6,6 +6,7 @@ import pytest
from modules.videos.models import Video, Channel
from modules.videos.video import lib as video_lib
+from wrolpi.common import get_wrolpi_config
from wrolpi.db import get_db_curs
from wrolpi.downloader import Download
from wrolpi.files.lib import refresh_files
@@ -160,14 +161,15 @@ def test_api_download(test_session, test_client, test_directory):
assert not download.settings.get('tag_names')
-def test_api_download_video_tags(test_session, test_client, tag_factory):
+@pytest.mark.asyncio
+async def test_api_download_video_tags(test_session, test_async_client, tag_factory):
"""A user can request Tags for a video being downloaded."""
- tag1 = tag_factory()
- tag2 = tag_factory()
+ tag1 = await tag_factory()
+ tag2 = await tag_factory()
content = dict(urls=['https://example.com/video1', ], downloader='video',
settings={'tag_names': [tag1.name, tag2.name]})
- request, response = test_client.post('/api/download', content=json.dumps(content))
+ request, response = await test_async_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
@@ -176,7 +178,9 @@ def test_api_download_video_tags(test_session, test_client, tag_factory):
assert download.settings['tag_names'] == [tag1.name, tag2.name]
-def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory, assert_video_search):
+@pytest.mark.asyncio
+async def test_search_videos_file(test_client, test_session, test_directory, video_with_search_factory,
+ assert_video_search):
"""Test that videos can be searched and that their order is by their textsearch rank."""
# These captions have repeated letters, so they will be higher in the ranking.
videos = [
@@ -193,37 +197,38 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit
# Repeated runs should return the same result
for _ in range(2):
# Only videos with a b are returned, ordered by the amount of b's
- assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
+ await assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
# Only two captions have e
- assert_video_search(search_str='e', assert_ids=[4, 1])
+ await assert_video_search(search_str='e', assert_ids=[4, 1])
# Only 2 contains 2
- assert_video_search(search_str='2', assert_ids=[2, ])
+ await assert_video_search(search_str='2', assert_ids=[2, ])
# Only two captions have d
- assert_video_search(search_str='d', assert_ids=[1, 2])
+ await assert_video_search(search_str='d', assert_ids=[1, 2])
# 5 can be gotten by it's title
- assert_video_search(search_str='5', assert_ids=[5, ])
+ await assert_video_search(search_str='5', assert_ids=[5, ])
# only video 1 has e and d
- assert_video_search(search_str='e d', assert_ids=[1, ])
+ await assert_video_search(search_str='e d', assert_ids=[1, ])
# video 1 and 4 have b and e, but 1 has more
- assert_video_search(search_str='b e', assert_ids=[1, 4])
+ await assert_video_search(search_str='b e', assert_ids=[1, 4])
# Check totals are correct even with a limit
- assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
+ await assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
-def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory):
+@pytest.mark.asyncio
+async def test_search_videos(test_session, video_factory, assert_video_search, simple_channel, tag_factory):
"""Search the Video table. This does not need to use a join with the File table."""
vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2',
channel_id=simple_channel.id)
vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
- tag1, tag2 = tag_factory(), tag_factory()
+ tag1, tag2 = await tag_factory(), await tag_factory()
vid1.add_tag(tag1.id)
@@ -237,22 +242,68 @@ def test_search_videos(test_session, video_factory, assert_video_search, simple_
assert test_session.query(Video).count() == 3
assert test_session.query(FileGroup).count() == 3
- assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime')
- assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime')
- assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2)
- assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2,
- offset=1)
- assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime',
- channel_id=simple_channel.id)
- assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name])
- assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
- assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
+ await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='published_datetime')
+ await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='-published_datetime')
+ await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id], order_by='-published_datetime', limit=2)
+ await assert_video_search(assert_total=3, assert_ids=[vid2.id, vid1.id], order_by='-published_datetime', limit=2,
+ offset=1)
+ await assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime',
+ channel_id=simple_channel.id)
+ await assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name])
+ await assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
+ await assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
# Only vid2 has both tags.
- assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name])
+ await assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name])
# No results, no total is returned from the DB.
- assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2)
+ await assert_video_search(assert_total=0, assert_ids=[], tag_names=[tag1.name], offset=2)
# Check all order_by.
for order_by in video_lib.VIDEO_ORDERS:
- assert_video_search(order_by=order_by)
+ await assert_video_search(order_by=order_by)
+
+
+@pytest.mark.asyncio
+async def test_delete_video(test_async_client, test_session, video_factory, test_download_manager):
+ """When a Video record is deleted, the FileGroup should be deleted. The URL is added to the
+ global skip list so the video is not downloaded again."""
+ video = video_factory(with_video_file=True, with_poster_ext='png')
+ video.file_group.url = 'https://example.com/video/1'
+ files = video.file_group.my_paths()
+ test_session.commit()
+
+ request, response = await test_async_client.delete(f'/api/videos/video/{video.id}')
+ assert response.status_code == HTTPStatus.NO_CONTENT
+
+ assert test_session.query(Video).count() == 0, 'Video was not deleted.'
+ assert test_session.query(FileGroup).count() == 0, 'Video FileGroup was not deleted.'
+ assert not any(i.exists() for i in files), 'All files should have been deleted.'
+ assert test_download_manager.is_skipped('https://example.com/video/1'), 'Video should not be downloaded again.'
+
+
+@pytest.mark.asyncio
+async def test_format_videos_description(test_async_client, test_session, test_directory, channel_factory, tag_factory):
+ channel = channel_factory(name='Channel Name')
+ tag = await tag_factory()
+ test_session.commit()
+
+ body = dict()
+ request, response = await test_async_client.post('/api/videos/tag_info', json=body)
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['videos_destination'] == 'videos'
+
+ body = dict(channel_id=channel.id)
+ request, response = await test_async_client.post('/api/videos/tag_info', json=body)
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['videos_destination'] == 'videos/Channel Name'
+
+ body = dict(channel_id=channel.id, tag_name=tag.name)
+ request, response = await test_async_client.post('/api/videos/tag_info', json=body)
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['videos_destination'] == 'videos/one/Channel Name'
+
+ get_wrolpi_config().videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s'
+ body = dict(channel_id=channel.id, tag_name=tag.name)
+ request, response = await test_async_client.post('/api/videos/tag_info', json=body)
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name'
diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py
index 00044b5e..a0ce7ed7 100644
--- a/modules/videos/test/test_common.py
+++ b/modules/videos/test/test_common.py
@@ -135,10 +135,12 @@ def test_generate_video_poster(video_file):
@pytest.mark.asyncio
-async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config):
+async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config,
+ tag_factory):
"""Importing the Channels' config should create any missing download records"""
channel1 = channel_factory(source_id='foo', url='https://example.com/channel1')
channel2 = channel_factory(source_id='bar', url='https://example.com/channel2')
+ tag = await tag_factory()
assert len(test_session.query(Channel).all()) == 2
test_session.commit()
@@ -228,8 +230,19 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
assert {i.url for i in channel1.downloads} == {'https://example.com/channel1'}
assert {i.url for i in channel2.downloads} == {'https://example.com/channel2', 'https://example.org'}
+ # A Channel with a `tag_name` is tagged.
+ update_channel_config(get_channels_config(),
+ channel1.source_id,
+ {'tag_name': tag.name})
+ import_channels_config()
+ channel1, channel2 = test_session.query(Channel).order_by(Channel.url).all()
+ assert channel1.tag == tag and channel1.tag_id == tag.id
+ assert channel2.tag is None and channel2.tag_id is None
-def test_import_channel_delete_missing_channels(test_session, channel_factory, test_channels_config):
+
+@pytest.mark.asyncio
+async def test_import_channel_delete_missing_channels(
+ test_async_client, test_session, channel_factory, test_channels_config):
"""The Channel import function deletes any Channels that are not in the config."""
# Create a DB and config with two Channels.
channel1 = channel_factory(source_id='foo')
@@ -268,32 +281,6 @@ def test_import_channel_delete_missing_channels(test_session, channel_factory, t
assert str(channel2.directory) not in test_channels_config.read_text()
-def test_check_for_video_corruption(video_file, test_directory):
- # The test video is not corrupt.
- assert common.check_for_video_corruption(video_file) is False
-
- # An empty file is corrupt.
- empty_file = test_directory / 'empty file.mp4'
- empty_file.touch()
- assert common.check_for_video_corruption(empty_file) is True
-
- # A video file must be complete.
- truncated_video = test_directory / 'truncated_video.mp4'
- with truncated_video.open('wb') as fh:
- fh.write(video_file.read_bytes()[:10000])
- assert common.check_for_video_corruption(truncated_video) is True
-
- # Check for specific ffprobe errors.
- with mock.patch('modules.videos.common.subprocess') as mock_subprocess:
- # `video_file` is ignored for these calls.
- mock_subprocess.run().stderr = b'Something\nInvalid NAL unit size'
- assert common.check_for_video_corruption(video_file) is True
- mock_subprocess.run().stderr = b'Something\nError splitting the input into NAL units'
- assert common.check_for_video_corruption(video_file) is True
- mock_subprocess.run().stderr = b'Some stderr is fine'
- assert common.check_for_video_corruption(video_file) is False
-
-
@pytest.mark.asyncio
async def test_ffprobe_json(test_async_client, video_file, corrupted_video_file):
content = await common.ffprobe_json(video_file)
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index d432591d..13597176 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -107,7 +107,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f
mock_video_extract_info, mock_video_process_runner, tag_factory):
"""A Video is tagged when Download record requires it."""
video_path = video_file_factory()
- tag1, tag2 = tag_factory(), tag_factory()
+ tag1, tag2 = await tag_factory(), await tag_factory()
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
mock_video_extract_info.return_value = copy(example_video_json)
diff --git a/modules/videos/test/test_models.py b/modules/videos/test/test_models.py
index f1681765..19a2a487 100644
--- a/modules/videos/test/test_models.py
+++ b/modules/videos/test/test_models.py
@@ -34,7 +34,7 @@ async def test_channel_move(test_async_client, test_session, test_directory, cha
foo.mkdir(parents=True)
# Move the Channel.
- await channel.move(foo, test_session)
+ await channel.move_channel(foo, test_session)
assert channel.directory == foo
assert channel.directory != (test_directory / 'New Channel Directory')
diff --git a/modules/videos/test/test_video.py b/modules/videos/test/test_video.py
index 5d3c5537..a081ea6b 100644
--- a/modules/videos/test/test_video.py
+++ b/modules/videos/test/test_video.py
@@ -29,27 +29,11 @@ def test_delete_video_no_channel(test_session, simple_video):
assert test_session.query(Video).count() == 0
-def test_delete_video(test_session, video_factory):
- """When a Video record is deleted, all referenced file records should be deleted."""
- video = video_factory(with_video_file=True, with_poster_ext='png')
- test_session.commit()
-
- assert video.video_path.is_file(), 'Video file was not created.'
- assert video.poster_path.is_file(), 'Video poster was not created.'
- assert test_session.query(Video).count() == 1, 'Video was not created.'
- assert test_session.query(FileGroup).count() == 1, 'Video file and poster file were not created.'
-
- video.delete()
- test_session.commit()
- assert test_session.query(Video).count() == 0, 'Video was not deleted.'
- assert test_session.query(FileGroup).count() == 0, 'Video files were not deleted.'
-
-
@pytest.mark.asyncio
async def test_delete_video_with_tag(test_async_client, test_session, video_factory, tag_factory):
"""You cannot delete a video with a tag."""
video = video_factory(with_video_file=True, with_poster_ext='png')
- tag = tag_factory()
+ tag = await tag_factory()
video.add_tag(tag.name)
test_session.commit()
@@ -63,7 +47,7 @@ async def test_delete_video_with_tag(test_async_client, test_session, video_fact
@pytest.mark.asyncio
-async def test_video_channel_refresh(test_session, test_directory, channel_factory, video_factory):
+async def test_video_channel_refresh(test_async_client, test_session, test_directory, channel_factory, video_factory):
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
# Create a video file in this channel's directory.
channel = channel_factory()
@@ -101,7 +85,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]}
entry = channel.info_json['entries'][0]
- tag = tag_factory()
+ tag = await tag_factory()
vid1 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',
@@ -150,7 +134,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto
@pytest.mark.asyncio
-async def test_delete_renamed_video(test_session, channel_factory, video_factory, tag_factory):
+async def test_delete_renamed_video(test_async_client, test_session, channel_factory, video_factory, tag_factory):
"""If duplicate Video's exist, delete all the videos that do not have the new title."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4'
@@ -158,7 +142,7 @@ async def test_delete_renamed_video(test_session, channel_factory, video_factory
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'Some new title'}]}
entry = channel.info_json['entries'][0]
- tag = tag_factory()
+ tag = await tag_factory()
vid1 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',
diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py
index 79775c1e..0a7f8fb0 100644
--- a/modules/videos/video/lib.py
+++ b/modules/videos/video/lib.py
@@ -164,8 +164,11 @@ def delete_videos(*video_ids: int, session: Session = None):
raise UnknownVideo('Could not find videos to delete')
logger.warning(f'Deleting {len(videos)} videos')
+ # Get all URLs, skip them once (so config isn't saved multiple times).
+ urls = list(filter(None, [i.file_group.url for i in videos]))
+ download_manager.add_to_skip_list(*urls)
for video in videos:
- video.delete(add_to_skip_list=True)
+ video.delete()
session.commit()
diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py
index 7058657f..f2806f26 100644
--- a/modules/videos/video/test/test_api.py
+++ b/modules/videos/video/test/test_api.py
@@ -9,7 +9,7 @@ from modules.videos.models import Video
from wrolpi.dates import now
-def test_get_video_prev_next(test_session, channel_factory, video_factory):
+def test_get_video_prev_next(test_async_client, test_session, channel_factory, video_factory):
"""
Test that the previous and next videos will be retrieved when fetching a video.
"""
@@ -69,7 +69,7 @@ def test_get_video_prev_next(test_session, channel_factory, video_factory):
raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e
-def test_get_video_prev_next_no_upload_date(test_session, video_factory, channel_factory):
+def test_get_video_prev_next_no_upload_date(test_async_client, test_session, video_factory, channel_factory):
"""
Previous and Next Videos should be those next to a Video file when videos do not have upload dates.
"""
@@ -153,7 +153,7 @@ async def test_wrol_mode(test_async_client, simple_channel, simple_video, wrol_m
test_download_manager, tag_factory):
"""Many methods are blocked when WROL Mode is enabled."""
channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
- tag = tag_factory()
+ tag = await tag_factory()
await wrol_mode_fixture(True)
diff --git a/modules/zim/conftest.py b/modules/zim/conftest.py
index 5309cf78..16096d5c 100644
--- a/modules/zim/conftest.py
+++ b/modules/zim/conftest.py
@@ -100,14 +100,14 @@ def zim_factory(test_session, zim_path_factory, test_directory):
@pytest.fixture
-def assert_zim_search(test_client, test_directory):
+def assert_zim_search(test_async_client, test_directory):
from http import HTTPStatus
- def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [],
- expected_status_code: int = HTTPStatus.OK):
+ async def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [],
+ expected_status_code: int = HTTPStatus.OK):
content = dict(search_str=search_str, tag_names=tag_names)
assert isinstance(zim_id, int), 'This fixture only supports Zim ids of type integer'
- request, response = test_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
+ request, response = await test_async_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
assert response.status_code == expected_status_code
result = response.json['zim']
diff --git a/modules/zim/lib.py b/modules/zim/lib.py
index 4e9a3936..7c1d6492 100644
--- a/modules/zim/lib.py
+++ b/modules/zim/lib.py
@@ -16,7 +16,7 @@ from modules.zim.kiwix import KIWIX_CATALOG
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
from wrolpi import flags
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
- register_refresh_cleanup, background_task, get_wrolpi_config
+ register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate
from wrolpi.db import get_db_session, optional_session, get_db_curs
from wrolpi.downloader import DownloadFrequency
from wrolpi.files.lib import refresh_files, split_file_name_words
@@ -124,14 +124,11 @@ def get_entries_tags(paths: List[str], zim_id: int):
def get_unique_paths(*paths: str) -> List[str]:
"""Return a new list which contains unique Zim paths."""
- new_paths = list()
- simple_paths = list()
- for path in paths:
- simple_path = path.replace('-', '').replace('_', '').replace('/', '').lower()
- if simple_path not in simple_paths:
- new_paths.append(path)
- simple_paths.append(simple_path)
- return new_paths
+ unique_paths = unique_by_predicate(
+ paths,
+ lambda i: i.replace('-', '').replace('_', '').replace('/', '').lower()
+ )
+ return list(unique_paths)
@optional_session
diff --git a/modules/zim/test/test_api.py b/modules/zim/test/test_api.py
index 9bf95580..49151037 100644
--- a/modules/zim/test/test_api.py
+++ b/modules/zim/test/test_api.py
@@ -8,16 +8,17 @@ from modules.zim.models import Zim
from wrolpi.downloader import Download
-def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim_search):
+@pytest.mark.asyncio
+async def test_zim_search(test_async_client, test_session, test_zim, tag_factory, assert_zim_search):
"""Entries can be tagged and retrieved using their containing Zim."""
- tag1, tag2 = tag_factory(), tag_factory()
+ tag1, tag2 = await tag_factory(), await tag_factory()
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
test_session.commit()
# Test searching with only `search_str`.
- assert_zim_search('one', 1, {
+ await assert_zim_search('one', 1, {
'path': test_zim.path,
'search': [
# Entry "One" contains a 'one' in the title, and the text.
@@ -27,13 +28,13 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim
],
'estimate': 2,
})
- assert_zim_search('two', 1, {
+ await assert_zim_search('two', 1, {
'path': test_zim.path,
'search': [{'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271},
{'path': 'home', 'headline': 'testing.\nOne\nTwo', 'rank': 0.06079271}],
'estimate': 2,
})
- assert_zim_search('item', 1, {
+ await assert_zim_search('item', 1, {
'path': test_zim.path,
'search': [{'path': 'one', 'headline': 'first item', 'rank': 0.06079271},
{'path': 'two', 'headline': 'second item', 'rank': 0.06079271}],
@@ -41,35 +42,35 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim
})
# Test searching with `search_str` and `tag_names`.
- assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={
+ await assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={
'search': [
# Only entry "One" is tagged with Tag 2.
{'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271},
],
'estimate': 1,
})
- assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={
+ await assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={
'search': [
# Only "One" is tagged with Tag 1 and contains "one".
{'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271},
],
'estimate': 1,
})
- assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={
+ await assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={
'search': [
# Only "Two" was tagged with Tag 1 and contains "two".
{'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271},
],
'estimate': 1,
})
- assert_zim_search('two', 1, tag_names=[tag2.name, ], expected={
+ await assert_zim_search('two', 1, tag_names=[tag2.name, ], expected={
# Two was not tagged with Tag 1.
'search': [],
'estimate': 0,
})
# Test search with only `tag_names`.
- assert_zim_search('', 1, tag_names=[tag1.name, tag2.name], expected={
+ await assert_zim_search('', 1, tag_names=[tag1.name, tag2.name], expected={
'search': [
# Return all entries that are tagged with the Tags.
{'path': 'one', 'headline': 'This is the first item', 'rank': 0.0, 'tag_names': ['one', 'two']},
@@ -78,9 +79,10 @@ def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim
})
-def test_entries_by_tag(test_client, test_session, test_zim, tag_factory):
+@pytest.mark.asyncio
+async def test_entries_by_tag(test_async_client, test_session, test_zim, tag_factory):
"""Entries can be tagged and retrieved using their containing Zim."""
- tag1, tag2 = tag_factory(), tag_factory()
+ tag1, tag2 = await tag_factory(), await tag_factory()
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
@@ -96,36 +98,38 @@ def test_entries_by_tag(test_client, test_session, test_zim, tag_factory):
assert [(i.path, i.title) for i in result] == [('one', 'One'), ]
-def test_zim_crud(test_client, test_session, test_directory, test_zim):
- request, response = test_client.get('/api/zim')
+@pytest.mark.asyncio
+async def test_zim_crud(test_async_client, test_session, test_directory, test_zim):
+ request, response = await test_async_client.get('/api/zim')
assert response.status_code == HTTPStatus.OK
assert len(response.json['zims']) == 1
assert response.json['zims'][0]['path'] == test_zim.path.name
- request, response = test_client.delete('/api/zim/1')
+ request, response = await test_async_client.delete('/api/zim/1')
assert response.status_code == HTTPStatus.NO_CONTENT
assert not test_zim.path.exists()
assert test_session.query(Zim).count() == 0
-def test_zim_tag_and_untag(test_client, test_session, test_zim, tag_factory):
+@pytest.mark.asyncio
+async def test_zim_tag_and_untag(test_async_client, test_session, test_zim, tag_factory):
"""Zim entries can be Tagged and Untagged."""
- tag1, tag2 = tag_factory(), tag_factory()
+ tag1, tag2 = await tag_factory(), await tag_factory()
test_session.commit()
# Entries can be tagged in the API.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
- request, response = test_client.post('/api/zim/tag', content=json.dumps(content))
+ request, response = await test_async_client.post('/api/zim/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
# Entries can be untagged in the API.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
- request, response = test_client.post('/api/zim/untag', content=json.dumps(content))
+ request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
# Cannot untag twice.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
- request, response = test_client.post('/api/zim/untag', content=json.dumps(content))
+ request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NOT_FOUND
diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py
index 7899f9fd..42fef7f0 100644
--- a/modules/zim/test/test_lib.py
+++ b/modules/zim/test/test_lib.py
@@ -51,14 +51,14 @@ async def test_zim_get_entry(test_async_client, test_session, zim_path_factory):
@pytest.mark.asyncio
async def test_zim_bad_entry(test_async_client, test_session, test_zim, tag_factory):
- tag1 = tag_factory()
+ tag1 = await tag_factory()
with pytest.raises(UnknownZimEntry):
test_zim.tag_entry(tag1.name, 'path does not exist')
@pytest.mark.asyncio
async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, tag_factory):
- tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
+ tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
@@ -70,7 +70,8 @@ async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, t
@pytest.mark.asyncio
async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory, tag_factory):
- tze = zim_factory('zim1').tag_entry(tag_factory().name, 'one')
+ tag = await tag_factory()
+ tze = zim_factory('zim1').tag_entry(tag.name, 'one')
# Test TagZimEntry.__repr__
assert repr(tze) == ""
@@ -78,7 +79,7 @@ async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory,
@pytest.mark.asyncio
async def test_zim_entries_with_tags(test_async_client, test_session, zim_factory, tag_factory):
"""Can fetch Zim Entries that have Tags. Can filter only those entries that are tagged on a specific Zim."""
- tag1, tag2 = tag_factory(), tag_factory()
+ tag1, tag2 = await tag_factory(), await tag_factory()
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
# Tag Zim1 with both tags, but Zim2 shares one of the tags and is returned when no Zim is passed.
zim1.tag_entry(tag1.name, 'one')
@@ -124,13 +125,14 @@ def test_get_unique_paths():
'A/Thomas_Harvey_Johnston']
-def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now):
+@pytest.mark.asyncio
+async def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now):
fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
config = tags.get_tags_config()
assert not config.tag_zims
# Tag three Zim entries.
- tag1, tag2 = tag_factory('Tag1'), tag_factory('Tag2')
+ tag1, tag2 = await tag_factory('Tag1'), await tag_factory('Tag2')
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
@@ -243,12 +245,13 @@ async def test_zim_download(test_session, kiwix_download_zim, test_directory, te
assert not flags.outdated_zims.is_set()
-def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config):
+@pytest.mark.asyncio
+async def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config):
"""When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim.
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
- tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
+ tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
# Tag outdated Zim entries.
zim1.tag_entry(tag1.name, 'home')
zim1.tag_entry(tag2.name, 'one')
diff --git a/wrolpi/common.py b/wrolpi/common.py
index e9fb2e9e..d5b1948b 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -14,6 +14,7 @@ import socket
import string
import sys
import tempfile
+import traceback
from asyncio import Task
from copy import deepcopy
from dataclasses import dataclass
@@ -160,6 +161,7 @@ __all__ = [
'ConfigFile',
'DownloadFileInfo',
'DownloadFileInfoLink',
+ 'LOGGING_CONFIG',
'ModelHelper',
'WROLPI_CONFIG',
'aiohttp_get',
@@ -180,7 +182,6 @@ __all__ = [
'compile_tsvector',
'cum_timer',
'date_range',
- 'LOGGING_CONFIG',
'disable_wrol_mode',
'download_file',
'enable_wrol_mode',
@@ -227,6 +228,7 @@ __all__ = [
'truncate_generator_bytes',
'truncate_object_bytes',
'tsvector',
+ 'unique_by_predicate',
'url_strip_host',
'walk',
'wrol_mode_check',
@@ -252,10 +254,6 @@ class ModelHelper:
def get_by_path(path, session) -> Base:
raise NotImplementedError('This model has not defined this method.')
- @staticmethod
- def find_by_paths(paths, session) -> List:
- raise NotImplementedError('This model has not defined this method.')
-
@staticmethod
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
"""Attempts to get a model instance by its id. Returns None if no instance can be found."""
@@ -274,9 +272,10 @@ class ModelHelper:
def do_model(file_group, session: Session):
raise NotImplementedError('This model has not defined this method.')
- def flush(self):
+ def flush(self, session: Session = None):
"""A convenience method which flushes this record using its DB Session."""
- if session := Session.object_session(self):
+ session = session or Session.object_session(self)
+ if session:
session.flush([self])
else:
raise RuntimeError(f'{self} is not in a session!')
@@ -746,6 +745,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
synchronous_after = after
async def after(*a, **kw):
+ if logger.isEnabledFor(logging.DEBUG):
+ traceback.print_stack()
+ logger.debug(f'run_after synchronous_after called for {after}')
return synchronous_after(*a, **kw)
def wrapper(func: callable):
@@ -756,6 +758,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
@wraps(func)
async def wrapped(*a, **kw):
results = await func(*a, **kw)
+ if logger.isEnabledFor(logging.DEBUG):
+ traceback.print_stack()
+ logger.debug(f'run_after async called for {func}')
coro = after(*args, **kwargs)
asyncio.ensure_future(coro)
return results
@@ -763,6 +768,9 @@ def run_after(after: callable, *args, **kwargs) -> callable:
@wraps(func)
def wrapped(*a, **kw):
results = func(*a, **kw)
+ if logger.isEnabledFor(logging.DEBUG):
+ traceback.print_stack()
+ logger.debug(f'run_after sync called for {func}')
coro = after(*args, **kwargs)
asyncio.ensure_future(coro)
return results
@@ -1741,6 +1749,35 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
return headlines
+async def search_other_estimates(tag_names: List[str]) -> dict:
+ """Estimate other things that are Tagged."""
+ from wrolpi.db import get_db_curs
+
+ if not tag_names:
+ return dict(
+ channel_count=0,
+ )
+
+ with get_db_curs() as curs:
+ stmt = '''
+ SELECT
+ COUNT(c.id)
+ FROM channel c
+ LEFT OUTER JOIN public.tag t on t.id = c.tag_id
+ WHERE t.name = %(tag_name)s
+ '''
+ # TODO handle multiple tags
+ params = dict(tag_name=tag_names[0])
+ curs.execute(stmt, params)
+
+ channel_count = curs.fetchone()[0]
+
+ others = dict(
+ channel_count=channel_count,
+ )
+ return others
+
+
def replace_file(file: pathlib.Path | str, contents: str | bytes, missing_ok: bool = False):
"""Rename `file` to a temporary path, write contents to `file`, then, delete temporary file only if successful.
The original file will be restored if any of these steps fail."""
@@ -1891,3 +1928,29 @@ def is_valid_hex_color(hex_color: str) -> bool:
def is_hardlinked(path: pathlib.Path) -> bool:
"""Returns True only if `path` has more than one link."""
return path.stat().st_nlink > 1
+
+
+def _default_predicate(i):
+ return i
+
+
+def unique_by_predicate(
+ iterable: list | tuple | set,
+ predicate: callable = None,
+) -> list | tuple | set:
+ """Finds only unique items in the provided iterable based on the predicate function. Order is preserved.
+
+ `predicate` defaults to the items themselves.
+
+ >>> unique_by_predicate([['apple', 'banana', 'pear', 'kiwi', 'plum', 'peach']], lambda i: len(i))
+ # [ 'apple', 'banana', 'pear' ]
+ >>> unique_by_predicate([Path('foo.txt'), Path('foo.mp4'), Path('bar.txt')], lambda i: i.stem)
+ # [ Path('foo.txt'), Path('bar.txt') ]
+ >>> unique_by_predicate([1, 1, 2, 3, 3, 3, 3, 4], None)
+ # [ 1, 2, 3, 4 ]
+ """
+ predicate = predicate or _default_predicate # Return an iterable based off the object itself by default.
+ seen = set()
+ unique_items = [i for i in iterable if not ((key := predicate(i)) in seen or seen.add(key))]
+ # Return the same type as the input iterable
+ return iterable.__class__(unique_items) # For other iterable types
diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py
index 2492e107..8d1c4257 100644
--- a/wrolpi/conftest.py
+++ b/wrolpi/conftest.py
@@ -16,7 +16,7 @@ import zipfile
from abc import ABC
from datetime import datetime
from itertools import zip_longest
-from typing import List, Callable, Dict, Sequence, Union, Coroutine
+from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional
from typing import Tuple, Set
from unittest import mock
from unittest.mock import MagicMock, AsyncMock
@@ -175,7 +175,7 @@ async def test_async_client(test_directory) -> SanicASGITestClient:
@pytest.fixture
-def test_download_manager_config(test_async_client, test_directory):
+def test_download_manager_config(test_async_client, test_directory) -> pathlib.Path:
with downloads_manager_config_context():
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/download_manager.yaml'
@@ -187,7 +187,7 @@ async def test_download_manager(
test_async_client,
test_session, # session is required because downloads can start without the test DB in place.
test_download_manager_config,
-):
+) -> DownloadManager:
# Needed to use signals in test app?
api_app.signalize()
@@ -200,7 +200,7 @@ async def test_download_manager(
@pytest.fixture
-def fake_now():
+def fake_now() -> Callable:
try:
set_test_now(datetime(2000, 1, 1))
yield set_test_now
@@ -210,17 +210,17 @@ def fake_now():
@pytest.fixture
-def successful_download():
+def successful_download() -> DownloadResult:
return DownloadResult(success=True)
@pytest.fixture
-def failed_download():
- return DownloadResult(error='pytest.fixture failed_download error', success=False)
+def failed_download() -> DownloadResult:
+ return DownloadResult(error='wrolpi.conftest failed_download error', success=False)
@pytest.fixture
-def assert_download_urls(test_session):
+def assert_download_urls(test_session) -> Callable[[Set[str]], None]:
def asserter(expected_urls: Set[str]):
downloads = test_session.query(Download).all()
urls = {i.url for i in downloads}
@@ -231,7 +231,7 @@ def assert_download_urls(test_session):
@pytest.fixture
-def assert_downloads(test_session):
+def assert_downloads(test_session) -> Callable[[List[Dict]], None]:
from wrolpi.test.common import assert_dict_contains
def asserter(expected: List[Dict]):
@@ -245,7 +245,7 @@ def assert_downloads(test_session):
@pytest.fixture
-def test_downloader(test_download_manager):
+def test_downloader(test_download_manager) -> Downloader:
class TestDownloader(Downloader, ABC):
"""A testing Downloader"""
name = 'test_downloader'
@@ -314,7 +314,7 @@ def corrupted_video_file(test_directory) -> pathlib.Path:
@pytest.fixture
-def video_file_factory(test_directory):
+def video_file_factory(test_directory) -> Callable[[pathlib.Path], pathlib.Path]:
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
def _(path: pathlib.Path = None) -> pathlib.Path:
@@ -327,7 +327,7 @@ def video_file_factory(test_directory):
@pytest.fixture
-def video_bytes():
+def video_bytes() -> bytes:
return (PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()
@@ -467,7 +467,7 @@ def wrol_mode_fixture(test_config, test_download_manager) -> Callable[[bool], Co
@pytest.fixture
-def mock_create_subprocess_shell():
+def mock_create_subprocess_shell() -> Callable:
def mocker(communicate_return=None, return_code=None, communicate_side_effect=None):
async def communicate(*a, **kw):
if communicate_side_effect:
@@ -507,14 +507,14 @@ def flags_lock():
@pytest.fixture()
-def tag_factory(test_session):
+def tag_factory(test_session) -> Callable[[Optional[str]], Awaitable[Tag]]:
names = ['one', 'two', 'three', 'four', 'five', 'six']
count = 1
- def factory(name: str = None) -> Tag:
+ async def factory(name: str = None) -> Tag:
if not name:
name = names.pop(0)
- tag = upsert_tag(name, f'#{str(count) * 6}')
+ tag = await upsert_tag(name, f'#{str(count) * 6}', session=test_session)
return tag
return factory
diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py
index aa460aa8..08959b5e 100644
--- a/wrolpi/contexts.py
+++ b/wrolpi/contexts.py
@@ -52,6 +52,7 @@ def attach_shared_contexts(app: Sanic):
# Locks
app.shared_ctx.config_save_lock = multiprocessing.Lock()
+ app.shared_ctx.events_lock = multiprocessing.Lock()
reset_shared_contexts(app)
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index c0636a23..0fbefa56 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -253,6 +253,15 @@ class Download(ModelHelper, Base): # noqa
return download
raise UnknownDownload(f'Cannot find Download with URL: {url}')
+ @classmethod
+ @optional_session
+ def get_all_by_destination(cls, destination: str | pathlib.Path, session: Session = None) -> List['Download']:
+ destination = str(destination)
+ downloads = session.query(Download) \
+ .filter(Download.settings.contains({'destination': destination})) \
+ .all()
+ return downloads
+
class Downloader:
name: str = None
@@ -1330,7 +1339,7 @@ class RSSDownloader(Downloader, ABC):
# Only download new URLs.
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(*urls)]]
- logger.info(f'Successfully got {len(urls)} urls from rss {download.url}')
+ logger.info(f'Successfully got {len(urls)} new URLs from RSS {download.url}')
return DownloadResult(success=True, downloads=urls)
diff --git a/wrolpi/events.py b/wrolpi/events.py
index 694d0302..7a55ddd8 100644
--- a/wrolpi/events.py
+++ b/wrolpi/events.py
@@ -10,7 +10,6 @@ from wrolpi.dates import now
logger = logger.getChild(__name__)
HISTORY_SIZE = 100
-EVENTS_LOCK = multiprocessing.Lock()
class Events:
@@ -79,6 +78,14 @@ class Events:
def send_shutdown_failed(message: str = None):
send_event('shutdown_failed', message, subject='shutdown')
+ @staticmethod
+ def send_file_move_completed(message: str = None):
+ send_event('file_move_completed', message, subject='refresh')
+
+ @staticmethod
+ def send_file_move_failed(message: str = None):
+ send_event('file_move_failed', message, subject='refresh')
+
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
log = f'{event=}'
@@ -93,7 +100,7 @@ def log_event(event: str, message: str = None, action: str = None, subject: str
def send_event(event: str, message: str = None, action: str = None, subject: str = None, url: str = None):
from wrolpi.api_utils import api_app
- EVENTS_LOCK.acquire()
+ api_app.shared_ctx.events_lock.acquire()
try:
# All events will be in time order, they should never be at the exact same time.
dt = now()
@@ -112,7 +119,7 @@ def send_event(event: str, message: str = None, action: str = None, subject: str
while len(api_app.shared_ctx.events_history) > HISTORY_SIZE:
api_app.shared_ctx.events_history.pop(0)
finally:
- EVENTS_LOCK.release()
+ api_app.shared_ctx.events_lock.release()
log_event(event, message, action, subject)
diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py
index 046bba9d..6b01c0cb 100644
--- a/wrolpi/files/api.py
+++ b/wrolpi/files/api.py
@@ -12,6 +12,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict
from . import lib, schema
from ..api_utils import json_response, api_app
+from ..events import Events
from ..schema import JSONErrorResponse
from ..tags import Tag
from ..vars import PYTEST
@@ -224,7 +225,9 @@ async def post_move(_: Request, body: schema.Move):
try:
await lib.move(destination, *sources)
+ Events.send_file_move_completed(f'Moving files to {body.destination} succeeded')
except Exception as e:
+ Events.send_file_move_failed(f'Moving files to {body.destination} failed!')
raise FileConflict(f'Failed to move {sources} to {destination}') from e
return response.empty(HTTPStatus.NO_CONTENT)
@@ -245,7 +248,7 @@ async def post_rename(_: Request, body: schema.Rename):
raise FileConflict(f'File already exists: {new_path}')
try:
- await lib.rename(path, body.new_name)
+ await lib.rename(path, body.new_name, send_events=True)
except Exception as e:
logger.error(f'Failed to rename {path} to {new_path}', exc_info=e)
raise FileConflict(f'Failed to rename {path} to {new_path}') from e
@@ -297,14 +300,15 @@ async def post_upload(request: Request):
Will not overwrite an existing file, unless a previous upload did not complete."""
try:
- destination = request.form['destination'][0]
+ destination_str = request.form['destination'][0]
except Exception as e:
logger.error(f'Failed to get upload destination', exc_info=e)
raise UnknownDirectory('Must provide destination') from e
- destination = get_media_directory() / destination
- if not destination.is_dir():
- raise UnknownDirectory(f'Destination must be a relative directory that is already in the media directory: {destination}')
+ destination = get_media_directory() / destination_str
+ if destination_str.startswith('/') or destination_str.startswith('.') or not destination.is_dir():
+ msg = f'Destination must be a relative directory that is already in the media directory: {destination}'
+ raise UnknownDirectory(msg)
try:
filename = str(request.form['filename'][0])
@@ -385,7 +389,11 @@ async def post_upload(request: Request):
logger.info(f'Got final chunk of uploaded file: {output_str}')
del api_app.shared_ctx.uploaded_files[output_str]
# Upsert this new file (and any related files) into the DB.
- background_task(lib.upsert_file(output, tag_names=tag_names))
+ coro = lib.upsert_file(output, tag_names=tag_names)
+ if PYTEST:
+ await coro
+ else:
+ background_task(coro)
return response.empty(HTTPStatus.CREATED)
# Request the next chunk.
diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py
index e6fb16a0..376cea53 100644
--- a/wrolpi/files/lib.py
+++ b/wrolpi/files/lib.py
@@ -9,13 +9,15 @@ import pathlib
import re
import shutil
import subprocess
+from collections import OrderedDict
from itertools import zip_longest
from pathlib import Path
-from typing import List, Tuple, Union, Dict, Generator, Iterable
+from typing import List, Tuple, Union, Dict, Generator, Iterable, Set
import cachetools.func
import psycopg2
from sqlalchemy import asc
+from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from wrolpi import flags
@@ -24,7 +26,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
partition, cancelable_wrapper, \
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \
get_wrolpi_config, \
- timer
+ timer, chunks, unique_by_predicate
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
@@ -32,7 +34,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
from wrolpi.events import Events
from wrolpi.files.models import FileGroup, Directory
from wrolpi.lang import ISO_639_CODES
-from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where
+from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config
from wrolpi.vars import PYTEST
try:
@@ -48,7 +50,7 @@ logger = logger.getChild(__name__)
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
- 'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file',
+ 'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
'move', 'rename', 'delete_directory', 'handle_file_group_search_results']
@@ -311,6 +313,12 @@ def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = Fals
return path.name, ''
+def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...] | Set[pathlib.Path]) \
+ -> List[pathlib.Path]:
+ """Returns the first of each Path with a unique stem. Used to detect if a group of files share a stem."""
+ return unique_by_predicate(files, lambda i: split_path_stem_and_suffix(i)[0])
+
+
refresh_logger = logger.getChild('refresh')
@@ -578,6 +586,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
refreshing_all_files = False
with flags.refreshing, timer('refresh_files', 'info'):
+ api_app.shared_ctx.refresh['counted_files'] = 0
if not paths:
refresh_logger.warning('Refreshing all files')
refreshing_all_files = True
@@ -603,8 +612,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
continue
directories.extend(dirs)
found_directories |= set(dirs)
- api_app.shared_ctx.refresh['counted_files'] = api_app.shared_ctx.refresh.get('counted_files',
- 0) + len(files)
+ add_files_to_refresh_count(len(files))
# Sleep to catch cancel.
await asyncio.sleep(0)
refresh_logger.info(f'Counted {api_app.shared_ctx.refresh["counted_files"]} files')
@@ -633,7 +641,7 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
with get_db_session() as session:
parent_directories = {i[0] for i in session.query(Directory.path).filter(Directory.path.in_(paths))}
parent_directories |= set(filter(lambda i: i.is_dir(), paths))
- await upsert_directories(parent_directories, found_directories)
+ upsert_directories(parent_directories, found_directories)
if send_events:
Events.send_global_after_refresh_completed()
@@ -684,7 +692,7 @@ async def apply_indexers():
break
-async def upsert_directories(parent_directories, directories):
+def upsert_directories(parent_directories, directories):
"""
Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in
`directories`.
@@ -1168,7 +1176,17 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
return 0
-def _move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]:
+MOVE_CHUNK_SIZE = 100
+
+
+def add_files_to_refresh_count(count: int):
+ from wrolpi.api_utils import api_app
+ count = api_app.shared_ctx.refresh.get('counted_files', 0) + count
+ api_app.shared_ctx.refresh['counted_files'] = count
+
+
+async def _move(destination: pathlib.Path, *sources: pathlib.Path, session: Session) \
+ -> OrderedDict[pathlib.Path, pathlib.Path]:
media_directory = get_media_directory()
for source in sources:
if not str(source).startswith(str(media_directory)):
@@ -1177,70 +1195,155 @@ def _move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathl
raise FileNotFoundError(f'{destination} is not within the media directory')
destination_existed = destination.is_dir()
+ destination.mkdir(parents=True, exist_ok=True)
- # [ (pathlib.Path('source'), pathlib.Path('destination')), ... ]
- plan = list()
- unique_paths = set()
- # Get all files that need to be moved. Check for destination conflicts.
- for source in sources:
- if source.is_dir():
- # Recursively move the directory's files.
- for path in walk(source):
- # Preserve the directory structure that contains this file.
- new_file = destination / source.name / path.relative_to(source)
- if new_file.exists():
- raise FileConflict(f'Cannot move {path} to {new_file} because it already exists')
- if path not in unique_paths:
- plan.append((path, new_file))
- unique_paths.add(path)
- elif source.is_file():
- # Move all files associated with the source file.
- for file in glob_shared_stem(source):
- new_file = destination / file.name
- if new_file.exists():
- raise FileConflict(f'Cannot move {file} to {new_file} because it already exists')
- if file not in unique_paths:
- plan.append((file, new_file))
- unique_paths.add(file)
+ # The files that will be moved. [ (old_file, new_file), ... ]
+ plan: Dict[pathlib.Path, pathlib.Path] = dict()
- logger.info(f'Executing move plan with {len(plan)} steps.')
+ def add_file_group_to_plan(file_group_: FileGroup):
+ primary_path_ = file_group_.primary_path
+ if primary_path_ not in plan:
+ new_primary_path_ = destination / primary_path_.name
+ if new_primary_path_.exists():
+ raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
+ plan[primary_path_] = new_primary_path_
- # Track the steps performed so they can be reversed.
- revert_plan = list()
+ def add_source_file_group_to_plan(file_group_: FileGroup, source_: pathlib.Path):
+ primary_path_ = file_group_.primary_path
+ if primary_path_ not in plan:
+ new_primary_path_ = destination / source_.name / primary_path_.relative_to(source)
+ if new_primary_path_.exists():
+ raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
+ plan[primary_path_] = new_primary_path_
- def do_plan(plan_):
- # Move deepest files first.
- plan_ = sorted(plan_, key=lambda i: (len(i[0].parents), i[0].name), reverse=True)
- for old_file, new_file in plan_: # noqa
- new_file.parent.mkdir(parents=True, exist_ok=True)
- logger.debug(f'Moving {old_file} to {new_file}')
- # if not new_file.exists():
- shutil.move(str(old_file), str(new_file))
- logger.debug(f'Moved {old_file} to {new_file}')
- revert_plan.append((new_file, old_file))
- # Move is complete.
- return plan_
+ # Sources may be a list of files, we only want to issue a query for one file in each FileGroup.
+ sources = get_unique_files_by_stem(sources)
+ logger.info(f'move got {len(sources)} to move')
- try:
- plan = do_plan(plan)
+ with flags.refresh_counting:
for source in sources:
- if source.is_dir():
- delete_directory(source)
- logger.info(f'Move execution completed')
- return plan
- except Exception as e:
- logger.error(f'Move failed', exc_info=e)
- # Move files back.
- do_plan(revert_plan)
- # Delete the directories that were created.
- directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
- for path in directories:
- if path.is_dir():
- delete_directory(path)
- # Remove destination only if it did not exist at the start.
- if destination.is_dir() and destination_existed is False:
- delete_directory(destination)
- raise
+ if source.is_file():
+ # Get any FileGroups that share the source's stem.
+ files = glob_shared_stem(source)
+ file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(files)).all()
+ if not file_groups:
+ # No FileGroups for this source file, create one.
+ try:
+ fg = FileGroup.from_paths(session, *files)
+ file_groups = [fg, ]
+ except NoPrimaryFile:
+ for file in files:
+ fg = FileGroup.from_paths(session, file)
+ add_file_group_to_plan(fg)
+ add_files_to_refresh_count(len(file_groups))
+ for fg in file_groups:
+ add_file_group_to_plan(fg)
+ # Sleep to catch cancel.
+ await asyncio.sleep(0)
+ elif source.is_dir():
+ stmt = 'SELECT primary_path FROM file_group WHERE primary_path LIKE :like'
+ for (primary_path,) in session.execute(stmt, dict(like=f'{source}%')):
+ primary_path = pathlib.Path(primary_path)
+ if primary_path not in plan:
+ new_primary_path = destination / source.name / primary_path.relative_to(source)
+ if new_primary_path.exists():
+ raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path}')
+ plan[primary_path] = new_primary_path
+ add_files_to_refresh_count(1)
+
+ # Sleep to catch cancel.
+ await asyncio.sleep(0)
+
+ # Move directories of this source.
+ for directory in (i for i in walk(source) if i.is_dir()):
+ new_directory = destination / source.name / directory.relative_to(source)
+ plan[directory] = new_directory
+
+ # Find any files in this directory that are not yet in the DB and add them to the plan.
+ files = {i for i in walk(source) if i.is_file()}
+ if missing_files := files - set(plan.keys()):
+ for paths in group_files_by_stem(missing_files):
+ try:
+ # Only create the FileGroup if a primary_path can be found.
+ get_primary_file(paths)
+ file_group = FileGroup.from_paths(session, *paths)
+ add_files_to_refresh_count(1)
+ add_source_file_group_to_plan(file_group, source)
+ except NoPrimaryFile:
+ # No primary path for these paths, create FileGroups for each.
+ add_files_to_refresh_count(1)
+ for file in paths:
+ fg = FileGroup.from_paths(session, file)
+ add_source_file_group_to_plan(fg, source)
+
+ # Sleep to catch cancel.
+ await asyncio.sleep(0)
+ else:
+ raise UnknownFile(f'Unknown path type, cannot move: {source}')
+
+ # Sort plan by the deepest files first.
+ plan: OrderedDict = OrderedDict(sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True))
+
+ # Revert plan is built out as files are moved.
+ revert_plan = OrderedDict()
+ new_directories = set()
+
+ def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path]):
+ """Apply the move plan to all the FileGroups.
+
+ @warning: Cannot be cancelled!"""
+ # Move FileGroups in groups.
+ for chunk in chunks(plan_.items(), MOVE_CHUNK_SIZE):
+ old_paths = [i for i, j in chunk]
+ old_files, old_directories = partition(lambda i: i.is_file(), old_paths)
+ file_groups_ = session.query(FileGroup).filter(FileGroup.primary_path.in_(old_files)).all()
+ if len(file_groups_) != len(old_files):
+ # We ensured there are FileGroups while building the plan, maybe user deleted a file?
+ raise RuntimeError('Could not get all FileGroups for move')
+
+ for file_group_ in file_groups_:
+ old_file = file_group_.primary_path
+ new_primary_path_ = plan_[old_file]
+ parent = new_primary_path_.parent
+ parent.mkdir(parents=True, exist_ok=True)
+ if parent not in new_directories:
+ new_directories.add(parent)
+ file_group_.move(new_primary_path_)
+ revert_plan[new_primary_path_] = old_file
+ for directory_ in old_directories:
+ delete_directory(directory_)
+
+ existing_directories = {i[0] for i in session.query(Directory.path)}
+ missing_directories = new_directories - existing_directories
+ for directory_ in missing_directories:
+ session.add(Directory(path=directory_, name=directory_.name))
+
+ # Don't forget what has moved.
+ session.flush(file_groups_)
+
+ with flags.refresh_discovery:
+ try:
+ do_plan(plan)
+ for source in sources:
+ if source.is_dir():
+ delete_directory(source)
+ logger.info(f'Move execution completed')
+ except Exception as e:
+ logger.error(f'Move failed', exc_info=e)
+ new_directories = set()
+ # Move files back. Get copy because do_plan will change the revert plan.
+ do_plan(revert_plan.copy())
+ # Delete the directories that were created.
+ directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
+ for path in directories:
+ if path.is_dir():
+ delete_directory(path)
+ # Remove destination only if it did not exist at the start.
+ if destination.is_dir() and destination_existed is False:
+ delete_directory(destination)
+ raise
+
+ return plan
def delete_directory(directory: pathlib.Path, recursive: bool = False):
@@ -1263,41 +1366,37 @@ def delete_directory(directory: pathlib.Path, recursive: bool = False):
curs.execute(stmt, (str(directory),))
-async def move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]:
+@optional_session
+async def move(destination: pathlib.Path, *sources: pathlib.Path, session: Session = None) \
+ -> OrderedDict[pathlib.Path, pathlib.Path]:
"""Moves a file or directory (recursively), preserving applied Tags.
If the move fails at any point, the files will be returned to their previous locations.
- Returns a list of tuples, the first Path in the tuple is the file's old location, the second Path is where the file
- was moved.
+ Returns an OrderedDict, the key is the file's old location, the value is where the file was moved.
"""
- # Move the files first, if this fails it will revert itself and raise an error.
- plan = _move(destination, *sources)
+ if not session:
+ raise RuntimeError('`session` is required')
- # Files were moved, migrate any Tags.
- plan_map = {old: new for old, new in plan}
- old_file_paths = list(plan_map.keys())
- with get_db_session(commit=True) as session:
- old_files: List[Tuple[FileGroup, TagFile]] = session.query(FileGroup, TagFile) \
- .filter(FileGroup.primary_path.in_(old_file_paths)) \
- .outerjoin(TagFile).all()
- logger.info(f'Moving {len(old_files)} tagged files')
- for old_file, _ in old_files:
- try:
- new_file_path = plan_map[old_file.primary_path]
- logger.info(f'Moving tagged file {old_file.primary_path} to {new_file_path}')
- old_file.move(new_file_path, move_files=False)
- except KeyError as e:
- # Got a duplicate?
- logger.error('Error during tag migration', exc_info=e)
- if PYTEST:
- raise
- session.commit()
+ with flags.refreshing:
+ from wrolpi.api_utils import api_app
+ api_app.shared_ctx.refresh['counted_files'] = 0
- if PYTEST:
- await refresh_files([destination, *sources])
- else:
- background_task(refresh_files([destination, *sources]))
+ # Move the files, if this fails it will revert itself and raise an error.
+ with timer('moving files', 'info'):
+ plan = await _move(destination, *sources, session=session)
+ session.commit()
+
+ with flags.refresh_indexing:
+ await apply_indexers()
+
+ with flags.refresh_modeling:
+ await apply_modelers()
+
+ with flags.refresh_cleanup:
+ await apply_refresh_cleanup()
+ # Save tags now that files have been moved.
+ save_tags_config()
return plan
@@ -1320,25 +1419,40 @@ async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
return new_path
-async def rename_directory(directory: pathlib.Path, new_name: str) -> pathlib.Path:
+async def rename_directory(directory: pathlib.Path, new_name: str, session: Session = None, send_events: bool = False) \
+ -> pathlib.Path:
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory."""
+ if not session:
+ raise RuntimeError('`session` is required')
+
new_directory = directory.with_name(new_name)
if new_directory.exists():
raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
# Move all paths into the new directory.
paths = list(directory.iterdir())
- await move(new_directory, *paths)
+ try:
+ await move(new_directory, *paths, session=session)
+ if send_events:
+ Events.send_file_move_completed(f'Directory has been renamed: {directory}')
+ except Exception:
+ if send_events:
+ Events.send_file_move_failed(f'Directory rename has failed: {directory}')
+ raise
# Remove the old directory.
delete_directory(directory)
return new_directory
-async def rename(path: pathlib.Path, new_name: str) -> pathlib.Path:
+@optional_session
+async def rename(path: pathlib.Path, new_name: str, session: Session = None, send_events: bool = False) -> pathlib.Path:
"""Rename a directory or file. Preserve any tags."""
+ if not session:
+ raise RuntimeError('`session` is required')
+
if path.is_dir():
- return await rename_directory(path, new_name)
+ return await rename_directory(path, new_name, session=session, send_events=send_events)
return await rename_file(path, new_name)
@@ -1399,26 +1513,36 @@ async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_nam
paths = glob_shared_stem(pathlib.Path(file))
# Remove any ignored files.
paths = remove_files_in_ignored_directories(paths)
- if paths:
- file_group = FileGroup.from_paths(session, *paths)
- logger.debug(f'upsert_file: {file_group}')
- # Re-index the contents of the file.
- session.add(file_group)
- session.flush([file_group, ])
-
- try:
- file_group.do_model(session)
- except Exception as e:
- logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
- if PYTEST:
- raise
-
- for tag_name in (tag_names or []):
- if tag_name not in file_group.tag_names:
- tag = Tag.get_by_name(tag_name)
- file_group.add_tag(tag.id)
-
- session.commit()
- return file_group
- else:
+ if not paths:
logger.warning('upsert_file called, but all files are in ignored directories!')
+ return
+
+ for i in range(2):
+ file_group = FileGroup.from_paths(session, *paths)
+ # Re-index the contents of the file.
+ try:
+ session.flush([file_group, ])
+ break
+ except IntegrityError:
+ # Another process inserted this FileGroup.
+ logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
+ session.rollback()
+ continue
+ else:
+ raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
+
+ logger.debug(f'upsert_file: {file_group}')
+ try:
+ file_group.do_model(session)
+ except Exception as e:
+ logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
+ if PYTEST:
+ raise
+
+ for tag_name in (tag_names or []):
+ if tag_name not in file_group.tag_names:
+ tag = Tag.get_by_name(tag_name)
+ file_group.add_tag(tag.id)
+
+ session.commit()
+ return file_group
diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py
index 0ca41891..da848ae9 100644
--- a/wrolpi/files/models.py
+++ b/wrolpi/files/models.py
@@ -10,7 +10,7 @@ from sqlalchemy import types
from sqlalchemy.orm import deferred, relationship, Session
from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
- get_relative_to_media_directory
+ get_relative_to_media_directory, unique_by_predicate
from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
from wrolpi.db import optional_session
from wrolpi.errors import FileGroupIsTagged, UnknownFile
@@ -154,9 +154,8 @@ class FileGroup(ModelHelper, Base):
"""Add all `paths` to this FileGroup.files."""
from wrolpi.files.lib import get_mimetype
new_files = list(self.files) if self.files else list()
- for file in paths:
- new_files.append(dict(path=file, mimetype=get_mimetype(file)))
- self.files = new_files
+ new_files.extend([dict(path=path, mimetype=get_mimetype(path)) for path in paths])
+ self.files = unique_by_predicate(new_files, lambda i: i['path'])
def my_files(self, *mimetypes: str) -> List[dict]:
"""Return all files related to this group that match any of the provided mimetypes.
@@ -225,7 +224,9 @@ class FileGroup(ModelHelper, Base):
if session := Session.object_session(self):
from wrolpi.downloader import download_manager
- if self.url and add_to_skip_list is True:
+ if self.url and (download := download_manager.get_download(session, self.url)):
+ download.delete(add_to_skip_list)
+ elif self.url and add_to_skip_list is True:
download_manager.add_to_skip_list(self.url)
session.delete(self)
@@ -254,25 +255,26 @@ class FileGroup(ModelHelper, Base):
@classmethod
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
"""Create a new FileGroup which contains the provided file paths."""
- from wrolpi.files.lib import get_primary_file, get_mimetype, split_path_stem_and_suffix
+ from wrolpi.files.lib import get_primary_file, get_mimetype, get_unique_files_by_stem
- unique_stems = {split_path_stem_and_suffix(i, full=True)[0] for i in paths}
+ unique_stems = get_unique_files_by_stem(paths)
if len(unique_stems) > 1:
raise RuntimeError(f'Refusing to create FileGroup with paths that do not share a stem.')
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
+ logger.debug(f'FileGroup.from_paths: {len(existing_groups)}')
if len(existing_groups) == 0:
# These paths have not been used previously, create a new FileGroup.
file_group = FileGroup()
primary_path = get_primary_file(paths)
file_group.primary_path = primary_path
file_group.append_files(*paths)
- file_group.mimetype = get_mimetype(file_group.primary_path)
session.add(file_group)
elif len(existing_groups) == 1:
# Found one FileGroup with these paths, no need to create a new FileGroup.
file_group = existing_groups[0]
- primary_path = file_group.primary_path
+ file_group.append_files(*paths)
+ primary_path = get_primary_file(file_group.my_paths())
else:
# Multiple FileGroups contain these paths as primary.
primary_path = get_primary_file(paths)
@@ -284,8 +286,11 @@ class FileGroup(ModelHelper, Base):
if not isinstance(primary_path, pathlib.Path):
raise ValueError('Cannot create FileGroup without a primary path.')
+ file_group.primary_path = primary_path
file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths))
file_group.size = sum(i.stat().st_size for i in paths)
+ file_group.mimetype = get_mimetype(file_group.primary_path)
+ logger.debug(f'FileGroup.from_paths: {file_group}')
return file_group
@@ -329,28 +334,44 @@ class FileGroup(ModelHelper, Base):
self.files = collected_files
- def move(self, new_primary_path: pathlib.Path, move_files: bool = True):
+ def move(self, new_primary_path: pathlib.Path):
"""Move all files in this group to a new location."""
- from wrolpi.files.lib import split_path_stem_and_suffix
- if move_files and new_primary_path.exists():
+ from wrolpi.files.lib import split_path_stem_and_suffix, glob_shared_stem
+
+ if new_primary_path.exists():
raise FileExistsError(f'Cannot move {self} to {new_primary_path} because it already exists.')
+ # Adopt any new files when a FileGroup has multiple files.
+ if len(self.my_files()) > 1:
+ self.append_files(*glob_shared_stem(self.primary_path))
+
new_name, _ = split_path_stem_and_suffix(new_primary_path, full=True)
# Need a deepcopy because changes to self.files are ignored otherwise.
new_files = copy.deepcopy(self.files)
+ # Ensure that all destination files do not yet exist before move.
+ for file in new_files:
+ _, suffix = split_path_stem_and_suffix(file['path'])
+ new_path = pathlib.Path(f'{new_name}{suffix}')
+ if new_path.exists():
+ raise FileExistsError(f'Cannot move {self} to {new_path} because it already exists.')
+
+ # Collect all files that exist and move them.
+ existing_files = list()
for idx, file in enumerate(new_files):
_, suffix = split_path_stem_and_suffix(file['path'])
new_path = pathlib.Path(f'{new_name}{suffix}')
- if move_files:
+ if pathlib.Path(file['path']).is_file():
shutil.move(file['path'], new_path)
- new_files[idx]['path'] = new_path
- self.files = new_files
+ existing_files.append({'path': new_path, 'mimetype': file['mimetype']})
+
+ logger.debug(f'Moved FileGroup: {self.primary_path} -> {new_primary_path}')
+ self.files = existing_files
if self.title == self.primary_path.name:
# Do not overwrite title from modeler.
self.title = new_primary_path.name
self.primary_path = new_primary_path
# Need to re-index for self.data.
- self.indexed = False
+ self.indexed = False if self.data else self.indexed
# Flush the changes to the FileGroup.
self.flush()
@@ -384,11 +405,15 @@ class FileGroup(ModelHelper, Base):
if model_class := self.get_model_class():
if model := model_class.get_by_path(self.primary_path, session):
# Already modeled.
+ logger.debug(f'already modeled: {self} {model}')
return model
# Create new model (Video/Archive/Ebook).
- return model_class.do_model(self, session)
+ model = model_class.do_model(self, session)
+ logger.debug(f'new model: {self} {model}')
+ return model
# Index the file if no models are available.
+ logger.debug(f'no model class: {self}')
self.do_index()
self.indexed = True
diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py
index a266b176..54a965d6 100644
--- a/wrolpi/files/test/test_api.py
+++ b/wrolpi/files/test/test_api.py
@@ -190,7 +190,7 @@ def test_files_search(test_session, test_client, make_files_structure, assert_fi
@pytest.mark.asyncio
async def test_files_search_any_tag(test_async_client, test_session, make_files_structure, tag_factory):
- one, two = tag_factory(), tag_factory()
+ one, two = await tag_factory(), await tag_factory()
files = [
'foo.txt',
'foo bar.txt',
@@ -299,34 +299,35 @@ def test_file_statistics(test_session, test_client, test_directory, example_pdf,
}
-def test_file_group_tag_by_primary_path(test_session, test_client, test_directory, example_singlefile, tag_factory,
- insert_file_group):
+@pytest.mark.asyncio
+async def test_file_group_tag_by_primary_path(test_session, test_async_client, test_directory, example_singlefile,
+ tag_factory, insert_file_group):
singlefile = FileGroup.from_paths(test_session, example_singlefile)
- tag1 = tag_factory()
- tag2 = tag_factory()
+ tag1 = await tag_factory()
+ tag2 = await tag_factory()
test_session.commit()
# FileGroup can be tagged with its primary_path.
content = dict(file_group_primary_path=str(singlefile.primary_path.relative_to(test_directory)), tag_name=tag1.name)
- request, response = test_client.post('/api/files/tag', content=json.dumps(content))
+ request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert test_session.query(TagFile).count() == 1
# FileGroup can be tagged with its id.
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
- request, response = test_client.post('/api/files/tag', content=json.dumps(content))
+ request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert test_session.query(TagFile).count() == 2
# FileGroup can be untagged with its primary_path.
content = dict(file_group_id=singlefile.id, tag_id=tag1.id)
- request, response = test_client.post('/api/files/untag', content=json.dumps(content))
+ request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(TagFile).count() == 1
# FileGroup can be untagged with its id.
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
- request, response = test_client.post('/api/files/untag', content=json.dumps(content))
+ request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(TagFile).count() == 0
@@ -468,11 +469,13 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f
assert test_session.query(FileGroup).count() == 1
-def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form, tag_factory,
- video_bytes):
+@pytest.mark.asyncio
+async def test_post_upload(test_session, test_async_client, test_directory, make_files_structure, make_multipart_form,
+ tag_factory,
+ video_bytes):
"""A file can be uploaded in chunks directly to the destination."""
make_files_structure(['uploads/'])
- tag1, tag2 = tag_factory(), tag_factory()
+ tag1, tag2 = await tag_factory(), await tag_factory()
part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:]
forms = [
@@ -487,7 +490,7 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
- request, response = test_client.post('/api/files/upload', content=body, headers=headers)
+ request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK, response.content.decode()
output = test_directory / 'uploads/video.mp4'
@@ -505,7 +508,7 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
dict(name='chunk', value=part2, filename='chunk'),
]
body = make_multipart_form(forms)
- request, response = test_client.post('/api/files/upload', content=body, headers=headers)
+ request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
assert output.is_file()
diff --git a/wrolpi/files/test/test_ebooks.py b/wrolpi/files/test/test_ebooks.py
index 143cb92b..2655c5f1 100644
--- a/wrolpi/files/test/test_ebooks.py
+++ b/wrolpi/files/test/test_ebooks.py
@@ -137,7 +137,7 @@ async def test_discover_calibre_cover(test_session, test_directory, example_epub
@pytest.mark.asyncio
async def test_move_ebook(test_async_client, test_session, test_directory, example_epub, image_file, tag_factory):
"""An ebook is re-indexed when moved."""
- tag = tag_factory()
+ tag = await tag_factory()
shutil.move(image_file, example_epub.with_suffix('.jpg'))
await refresh_files()
diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py
index 9a79584f..4ca75dc4 100644
--- a/wrolpi/files/test/test_lib.py
+++ b/wrolpi/files/test/test_lib.py
@@ -82,7 +82,7 @@ async def test_delete_file_names(test_async_client, test_session, make_files_str
})
foo_fg = FileGroup.from_paths(test_session, foo)
foo1_fg = FileGroup.from_paths(test_session, foo1)
- tag = tag_factory()
+ tag = await tag_factory()
foo1_fg.add_tag(tag.id)
test_session.commit()
assert foo.is_file()
@@ -106,7 +106,7 @@ async def test_delete_file_link(test_async_client, test_session, test_directory)
@pytest.mark.asyncio
async def test_delete_tagged(test_async_client, test_session, make_files_structure, tag_factory, video_bytes):
"""Cannot delete a file that has been tagged."""
- tag = tag_factory()
+ tag = await tag_factory()
make_files_structure({'foo/bar.txt': 'asdf', 'foo/bar.mp4': video_bytes})
await lib.refresh_files()
# Both files end up in a group.
@@ -365,7 +365,7 @@ async def test_file_group_tag(test_session, make_files_structure, test_directory
"""A FileGroup can be tagged."""
make_files_structure(['foo.mp4'])
await lib.refresh_discover_paths([test_directory, ], now())
- one = tag_factory()
+ one = await tag_factory()
foo: FileGroup = test_session.query(FileGroup).one()
foo.add_tag(one.id)
@@ -401,7 +401,7 @@ async def test_refresh_many_files(test_async_client, test_session, make_files_st
@pytest.mark.asyncio
-async def test_refresh_cancel(test_session, make_files_structure, test_directory):
+async def test_refresh_cancel(test_async_client, test_session, make_files_structure, test_directory):
"""Refresh tasks can be canceled."""
# Creat a lot of files so the refresh will take too long.
make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)])
@@ -723,15 +723,16 @@ async def test_refresh_directories(test_session, test_directory, assert_director
assert_directories({'foo', 'baz'})
-def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory,
- video_bytes, srt_file3):
+@pytest.mark.asyncio
+async def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory,
+ video_bytes, srt_file3):
"""A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be
migrated."""
vid, srt = make_files_structure({
'vid.mp4': video_bytes,
'vid.srt': (PROJECT_DIR / 'test/example3.en.srt').read_text(),
})
- one, two = tag_factory(), tag_factory()
+ one, two = await tag_factory(), await tag_factory()
vid_group = FileGroup.from_paths(test_session, vid)
srt_group = FileGroup.from_paths(test_session, srt)
test_session.add_all([vid_group, srt_group])
@@ -774,10 +775,11 @@ async def test_move(test_async_client, test_session, test_directory, make_files_
})
foo = test_directory / 'foo'
qux = test_directory / 'qux'
+ await lib.refresh_files()
# mv foo qux
- plan = await lib.move(qux, foo)
- plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan]
+ plan = await lib.move(qux, foo, session=test_session)
+ plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan.items()]
# The deepest files are moved first.
assert plan == [('foo/bar/baz/archive.html', 'qux/foo/bar/baz/archive.html'),
('foo/bar/video.mp4', 'qux/foo/bar/video.mp4'),
@@ -805,10 +807,16 @@ async def test_move_files(test_async_client, test_session, test_directory, make_
})
dest = test_directory / 'dest'
- await lib.move(dest, one, two)
+ plan = await lib.move(dest, one, two, session=test_session)
+ assert list(plan.items()) == [
+ (test_directory / 'foo/one.txt', test_directory / 'dest/one.txt'),
+ (test_directory / 'two.txt', test_directory / 'dest/two.txt'),
+ ]
# one.txt is moved out of foo.
- assert (dest / 'one.txt').read_text() == 'one'
- assert (dest / 'two.txt').read_text() == 'two'
+ assert not (test_directory / 'foo/one.txt').exists(), 'one.txt is lingering in source'
+ assert not (test_directory / 'two.txt').exists(), 'two.txt is lingering in source'
+ assert (dest / 'one.txt').read_text() == 'one', 'one.txt was not moved to destination'
+ assert (dest / 'two.txt').read_text() == 'two', 'two.txt was not moved to destination'
@pytest.mark.asyncio
@@ -830,8 +838,14 @@ async def test_move_deep_directory(test_async_client, test_session, test_directo
bar, qux = baz.parent, quux.parent
dest = test_directory / 'deep/dest'
- # mv foo/bar foo/qux foo/quuz.mp4 deep/dest
- await lib.move(dest, bar, qux, quuz_mp4)
+ # mv foo/bar foo/qux foo/quuz.mp4 foo/quuz.txt deep/dest
+ plan = await lib.move(dest, bar, qux, quuz_mp4, quuz_txt, session=test_session)
+ assert list(plan.items()) == [
+ (test_directory / 'foo/qux/quux.txt', test_directory / 'deep/dest/qux/quux.txt'),
+ (test_directory / 'foo/bar/baz.txt', test_directory / 'deep/dest/bar/baz.txt'),
+ (test_directory / 'foo/quuz.txt', test_directory / 'deep/dest/quuz.txt'),
+ (test_directory / 'foo/quuz.mp4', test_directory / 'deep/dest/quuz.mp4'),
+ ]
# Unrelated files and directories are untouched.
assert not (test_directory / 'deep/dest/foo').is_dir(), 'foo/ should not have been moved.'
@@ -861,7 +875,7 @@ async def test_move_directory(test_async_client, test_session, test_directory, m
assert_directories({'foo'})
bar = test_directory / 'bar'
- await lib.rename(test_directory / 'foo', 'bar')
+ await lib.rename(test_directory / 'foo', 'bar', session=test_session)
assert (bar / 'one.txt').read_text() == 'one'
assert_directories({'bar'})
@@ -890,7 +904,7 @@ async def test_move_error(test_session, test_directory, make_files_structure, vi
return shutil.move(*args, **kwargs)
with mock.patch('wrolpi.files.lib.shutil.move', mock_shutil_move), pytest.raises(FileNotFoundError):
- await lib.move(qux, foo)
+ await lib.move(qux, foo, session=test_session)
# The move failed, the files should be moved back.
# foo was not deleted.
assert foo.is_dir()
@@ -923,21 +937,19 @@ async def test_file_group_move(test_async_client, test_session, make_files_struc
assert new_srt.read_text() == srt_text
assert not video.is_file()
assert not srt.is_file()
- # Moved files must be re-indexed.
- assert file_group.indexed is False
@pytest.mark.asyncio
async def test_move_tagged(test_async_client, test_session, test_directory, make_files_structure, tag_factory):
"""A FileGroup's tag is preserved when moved or renamed."""
- tag = tag_factory()
+ tag = await tag_factory()
foo, bar, = make_files_structure({
'foo/foo.txt': 'foo',
'foo/bar.txt': 'bar',
})
await lib.refresh_files()
bar_file_group, foo_file_group = test_session.query(FileGroup).order_by(FileGroup.primary_path)
- foo_file_group.title = 'custom title' # Should not be overwritten.
+ bar_file_group.title = 'custom title' # Should not be overwritten.
bar_file_group.add_tag(tag.id)
test_session.commit()
@@ -945,7 +957,7 @@ async def test_move_tagged(test_async_client, test_session, test_directory, make
qux.mkdir()
# Move both files into qux. The Tag should also be moved.
- await lib.move(qux, bar, foo)
+ await lib.move(qux, bar, foo, session=test_session)
new_foo = qux / 'foo.txt'
new_bar = qux / 'bar.txt'
# Files were moved.
@@ -963,10 +975,10 @@ async def test_move_tagged(test_async_client, test_session, test_directory, make
assert not foo_file_group.tag_files
assert foo_file_group.primary_path.is_file()
assert foo_file_group.primary_path == new_foo == foo_file_group.files[0]['path']
- assert foo_file_group.title == 'custom title', 'Custom title should not have been overwritten.'
+ assert bar_file_group.title == 'custom title', 'Custom title should not have been overwritten.'
# Rename "bar.txt" to "baz.txt"
- await lib.rename(new_bar, 'baz.txt')
+ await lib.rename(new_bar, 'baz.txt', session=test_session)
baz = new_bar.with_name('baz.txt')
assert baz.read_text() == 'bar'
@@ -1111,3 +1123,21 @@ async def test_upsert_file_video_with_channel(test_async_client, test_session, t
assert video.channel_id == channel.id, "Videos should be assigned to the Channel their files are in"
assert video.video_path, 'Video file was upserted'
assert video.info_json_path, 'Info json file not found near video file.'
+
+
+@pytest.mark.asyncio
+async def test_move_many_files(test_async_client, test_session, test_directory, make_files_structure):
+ make_files_structure([f'foo/{i}.txt' for i in range(10_000)])
+ await lib.refresh_files()
+
+ foo = test_directory / 'foo'
+ bar = test_directory / 'bar'
+
+ with timer('test_move_many_files'):
+ # mv foo bar
+ await lib.move(bar, foo, session=test_session)
+
+ assert not foo.is_dir()
+ assert bar.is_dir()
+ assert (bar / 'foo').is_dir()
+ assert (bar / 'foo/0.txt').is_file()
diff --git a/wrolpi/files/test/test_models.py b/wrolpi/files/test/test_models.py
new file mode 100644
index 00000000..78ee7944
--- /dev/null
+++ b/wrolpi/files/test/test_models.py
@@ -0,0 +1,21 @@
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_file_group_move(test_async_client, test_session, test_directory, video_factory):
+ """Any new or renamed files are adopted during FileGroup.move."""
+ video = video_factory(title='video', with_poster_ext='.webp')
+ test_session.commit()
+
+ # Move one of the FileGroup's files. It should be moved as well.
+ video.poster_path.rename(video.poster_path.with_suffix('.jpg'))
+
+ # Move the FilGroup.
+ new_primary_path = video.video_path.with_name('new name')
+ video.file_group.move(new_primary_path)
+
+ # New file is discovered and adopted, missing file is removed.
+ assert video.file_group.my_paths() == [
+ (test_directory / 'videos/NO CHANNEL/new name.jpg'),
+ (test_directory / 'videos/NO CHANNEL/new name.mp4'),
+ ]
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index e0ffb19b..4c261d47 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -23,7 +23,7 @@ from wrolpi.admin import HotspotStatus
from wrolpi.api_utils import json_response, json_error_handler, api_app
from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_media_directory, \
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, \
- set_global_log_level, get_relative_to_media_directory
+ set_global_log_level, get_relative_to_media_directory, search_other_estimates
from wrolpi.dates import now
from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \
@@ -31,6 +31,7 @@ from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \
from wrolpi.events import get_events, Events
from wrolpi.files import files_bp
from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
+from wrolpi.tags import Tag
from wrolpi.vars import DOCKERIZED, IS_RPI, IS_RPI4, IS_RPI5, API_HOST, API_PORT, API_WORKERS, API_DEBUG, \
API_ACCESS_LOG, truthy_arg, API_AUTO_RELOAD
from wrolpi.version import __version__
@@ -118,10 +119,16 @@ async def index(_):
@openapi.description('Echo whatever is sent to this.')
@openapi.response(HTTPStatus.OK, schema.EchoResponse)
async def echo(request: Request):
+ try:
+ request_json = request.json
+ except Exception as e:
+ logger.error('Failed to parse request JSON', exc_info=e)
+ request_json = None
+ request_headers = dict(request.headers)
ret = dict(
form=request.form,
- headers=dict(request.headers),
- json=request.json,
+ headers=request_headers,
+ json=request_json,
method=request.method,
args=request.args,
)
@@ -137,7 +144,7 @@ def get_settings(_: Request):
ignored_directories = [get_relative_to_media_directory(i) for i in config.ignored_directories]
settings = {
- 'archive_directory': config.archive_destination,
+ 'archive_destination': config.archive_destination,
'download_manager_disabled': download_manager.is_disabled,
'download_manager_stopped': download_manager.is_stopped,
'download_on_startup': config.download_on_startup,
@@ -150,15 +157,15 @@ def get_settings(_: Request):
'ignore_outdated_zims': config.ignore_outdated_zims,
'ignored_directories': ignored_directories,
'log_level': api_app.shared_ctx.log_level.value,
- 'map_directory': config.map_destination,
+ 'map_destination': config.map_destination,
'nav_color': config.nav_color,
'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative.
'throttle_on_startup': config.throttle_on_startup,
'throttle_status': admin.throttle_status().name,
'version': __version__,
- 'videos_directory': config.videos_destination,
+ 'videos_destination': config.videos_destination,
'wrol_mode': config.wrol_mode,
- 'zims_directory': config.zims_destination,
+ 'zims_destination': config.zims_destination,
}
return json_response(settings)
@@ -482,7 +489,7 @@ async def get_tags_request(_: Request):
summary='Create or update a Tag',
)
async def post_tag(_: Request, body: schema.TagRequest, tag_id: int = None):
- tags.upsert_tag(body.name, body.color, tag_id)
+ await tags.upsert_tag(body.name, body.color, tag_id)
if tag_id:
return response.empty(HTTPStatus.OK)
else:
@@ -491,7 +498,7 @@ async def post_tag(_: Request, body: schema.TagRequest, tag_id: int = None):
@api_bp.delete('/tag/')
async def delete_tag_request(_: Request, tag_id: int):
- tags.delete_tag(tag_id)
+ Tag.find_by_id(tag_id).delete()
return response.empty()
@@ -594,4 +601,13 @@ async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimate
return json_response(ret)
+@api_bp.post('/search_other_estimates')
+@validate(json=schema.SearchOtherEstimateRequest)
+async def post_search_other_estimates(_: Request, body: schema.SearchOtherEstimateRequest):
+ """Used by the Global search to suggest FileGroup count to the user."""
+ others = await search_other_estimates(body.tag_names)
+ ret = dict(others=others)
+ return json_response(ret)
+
+
api_app.error_handler.add(Exception, json_error_handler)
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index 2fadb3c5..00d18e1a 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -206,3 +206,8 @@ class SearchFileEstimateRequest:
def __post_init__(self):
if self.any_tag and self.tag_names:
raise ValidationError('Cannot search for any tag, and list of tags.')
+
+
+@dataclass
+class SearchOtherEstimateRequest:
+ tag_names: List[str] = field(default_factory=lambda: list())
diff --git a/wrolpi/tags.py b/wrolpi/tags.py
index a824d538..0d60bfff 100644
--- a/wrolpi/tags.py
+++ b/wrolpi/tags.py
@@ -54,7 +54,9 @@ class Tag(ModelHelper, Base):
color = Column(String)
tag_files: List[TagFile] = relationship('TagFile', back_populates='tag', cascade='all')
- tag_zim_entries: List[TagFile] = relationship('TagZimEntry', back_populates='tag', cascade='all')
+ tag_zim_entries: List = relationship('TagZimEntry', back_populates='tag', cascade='all')
+ channels: List = relationship('Channel', primaryjoin='Tag.id==Channel.tag_id', back_populates='tag',
+ cascade='all')
def __repr__(self):
name = self.name
@@ -148,27 +150,71 @@ class Tag(ModelHelper, Base):
def has_relations(self) -> bool:
"""Returns True if this Tag has been used with any FileGroups or Zim Entries."""
- return bool(self.tag_files or self.tag_zim_entries)
+ return bool(any(self.tag_files) or any(self.tag_zim_entries) or any(self.channels))
- def update_tag(self, name: str, color: str, session: Session):
+ async def update_tag(self, name: str, color: str | None, session: Session):
"""Change name/color of tag. Ensures safeness of new values, checks for conflicts."""
if ',' in name:
raise InvalidTag('Tag name cannot have comma')
- if not is_valid_hex_color(color):
+ if color and not is_valid_hex_color(color):
raise InvalidTag('Tag color is invalid')
- # Replace forward-slash (linux directories) with unicode Big Solidus (U+29F8)
+ # Replace forward-slash (linux directories) with Unicode Big Solidus (U+29F8)
name = name.replace('/', '⧸')
name = INVALID_FILE_CHARS.sub('', name)
- if Tag.get_by_name(name, session):
+ if (other := self.get_by_name(name, session)) and other.id != self.id:
raise InvalidTag('Tag name already taken')
if flags.refreshing.is_set():
raise RefreshConflict('Refusing to rename tag while file refresh is in progress')
+ old_name = self.name
self.name = name
- self.color = color
+ self.color = color or self.color
+ if Session.object_session(self):
+ self.flush()
+
+ from modules.videos.models import Channel
+ from modules.videos.errors import ChannelDirectoryConflict
+
+ async def _():
+ for channel in self.channels:
+ channel: Channel
+ possible_directory = channel.format_directory(old_name)
+ if channel.directory == possible_directory:
+ # Channel is in this Tag's old directory, move the Channel to the new directory.
+ new_directory = channel.format_directory(name)
+ try:
+ new_directory.mkdir(parents=True)
+ except FileExistsError:
+ raise ChannelDirectoryConflict(f'Channel directory already exists: {new_directory}')
+
+ # Move the files of the Channel.
+ await channel.move_channel(new_directory, session)
+ else:
+ msg = f"Not moving Channel because it is not in this Tag's old directory:" \
+ f" {channel} {possible_directory} {self}"
+ logger.warning(msg)
+
+ if PYTEST:
+ await _()
+ else:
+ background_task(_())
+
+ def delete(self):
+ """Deletes this Tag, if it is unused.
+
+ @warning: Commits and saves configs"""
+ if self.has_relations():
+ raise UsedTag(f'Cannot delete {self.name} it is used')
+
+ session = Session.object_session(self)
+ session.delete(self)
+ session.commit()
+
+ save_tags_config(session)
+ sync_tags_directory(session)
class TagsConfig(ConfigFile):
@@ -394,16 +440,17 @@ def get_tags() -> List[dict]:
@optional_session
@run_after(save_tags_config)
-def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = None) -> Tag:
+async def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = None) -> Tag:
if tag_id:
tag = Tag.find_by_id(tag_id, session)
- tag.update_tag(name, color, session)
- tag.flush()
+ await tag.update_tag(name, color, session)
else:
tag = Tag()
- tag.update_tag(name, color, session)
+ await tag.update_tag(name, color, session)
session.add(tag)
+ tag.flush()
+
try:
session.commit()
except IntegrityError as e:
@@ -411,26 +458,12 @@ def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = Non
session.rollback()
raise InvalidTag(f'Name already taken') from e
- save_tags_config()
- sync_tags_directory()
+ save_tags_config(session)
+ sync_tags_directory(session)
return tag
-@optional_session
-def delete_tag(tag_id: int, session: Session = None):
- tag = Tag.find_by_id(tag_id, session)
-
- if tag.has_relations():
- raise UsedTag(f'Cannot delete {tag.name} it is used')
-
- session.delete(tag)
- session.commit()
-
- save_tags_config()
- sync_tags_directory()
-
-
@optional_session
def import_tags_config(session: Session = None):
"""Reads the Tags and TagFiles from the config file, upserts them in the DB."""
diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py
index f3db63b6..8dbb5e2c 100644
--- a/wrolpi/test/test_common.py
+++ b/wrolpi/test/test_common.py
@@ -901,3 +901,18 @@ def test_html_screenshot(singlefile_contents_factory):
])
def test_is_valid_hex_color(color, expected):
assert common.is_valid_hex_color(color) == expected
+
+
+@pytest.mark.parametrize('iterable,predicate,expected', [
+ ((1, 2, 2, 3, 4), lambda i: i, (1, 2, 3, 4)),
+ ([1, 1, 2, 3, 3, 3, 3, 4], None, [1, 2, 3, 4]),
+ ([1, 1, 2, 2, 3, 4, 5, 6, 7, 8], lambda i: i % 2, [1, 2]),
+ (['apple', 'banana', 'pear', 'kiwi', 'plum', 'peach'], lambda i: len(i), ['apple', 'banana', 'pear']),
+ (
+ [pathlib.Path('foo.txt'), pathlib.Path('foo.mp4'), pathlib.Path('bar.txt')],
+ lambda i: i.stem,
+ [pathlib.Path('foo.txt'), pathlib.Path('bar.txt')]
+ ),
+])
+def test_unique_by_predicate(iterable, predicate, expected):
+ assert common.unique_by_predicate(iterable, predicate) == expected
diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py
index b1148227..0368ab95 100644
--- a/wrolpi/test/test_downloader.py
+++ b/wrolpi/test/test_downloader.py
@@ -76,7 +76,7 @@ async def test_create_downloads(test_async_client, test_session, test_download_m
@pytest.mark.asyncio
async def test_recreate_download(test_session, test_download_manager, test_downloader, tag_factory):
"""Settings are preserved when a Download is restarted."""
- tag = tag_factory()
+ tag = await tag_factory()
settings = {'tag_names': [tag.name, ]}
# Create download with settings initially.
diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py
index 91c0aa82..6a501df0 100644
--- a/wrolpi/test/test_root_api.py
+++ b/wrolpi/test/test_root_api.py
@@ -392,7 +392,7 @@ async def test_get_status(test_async_client, test_session):
@pytest.mark.asyncio
async def test_download_crud(test_session, test_async_client, test_download_manager_config, tag_factory):
"""Test creating once-downloads and recurring downloads."""
- tag1, tag2 = tag_factory(), tag_factory()
+ tag1, tag2 = await tag_factory(), await tag_factory()
async def dispatch_downloads(*a, **kw):
pass
@@ -567,7 +567,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa
request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_)
assert response_.status_code == HTTPStatus.OK
- tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
+ tag1, tag2, tag3 = await tag_factory(), await tag_factory(), await tag_factory()
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
@@ -629,7 +629,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa
async def test_search_file_estimates_any_tag(test_session, test_async_client, archive_factory, tag_factory,
video_factory):
"""File estimates can be filtered by any_tag."""
- tag1 = tag_factory()
+ tag1 = await tag_factory()
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
archive_factory(domain='bar.com', contents='contents of bar bunny')
test_session.commit()
@@ -706,6 +706,12 @@ def test_recursive_errors():
@pytest.mark.asyncio
async def test_settings_special_directories(test_async_client, test_config):
"""Maintainer can change special directories."""
+ request, response = await test_async_client.get('/api/settings')
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['archive_destination'] == 'archive/%(domain)s'
+ assert response.json['map_destination'] == 'map'
+ assert response.json['videos_destination'] == 'videos/%(channel_tag)s/%(channel_name)s'
+ assert response.json['zims_destination'] == 'zims'
data = {'archive_destination': 'archives'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
@@ -731,3 +737,23 @@ async def test_settings_special_directories(test_async_client, test_config):
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
assert get_wrolpi_config().archive_destination == 'archive/%(domain)s'
+
+
+@pytest.mark.asyncio
+async def test_search_other_estimates(test_async_client, test_session, channel_factory, tag_factory):
+ # Can search without Tags or Channels.
+ body = dict(tag_names=[])
+ request, response = await test_async_client.post('/api/search_other_estimates', json=body)
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['others']['channel_count'] == 0, 'Only one Channel is tagged.'
+
+ tag = await tag_factory()
+ channel1 = channel_factory(tag_name=tag.name)
+ channel2 = channel_factory()
+ test_session.commit()
+
+ # One Channel is tagged.
+ body = dict(tag_names=[tag.name, ])
+ request, response = await test_async_client.post('/api/search_other_estimates', json=body)
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.'
diff --git a/wrolpi/test/test_tags.py b/wrolpi/test/test_tags.py
index 2356bee6..3e3fa547 100644
--- a/wrolpi/test/test_tags.py
+++ b/wrolpi/test/test_tags.py
@@ -6,7 +6,7 @@ import yaml
from wrolpi import tags
from wrolpi.common import is_hardlinked, walk
-from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, FileGroupAlreadyTagged
+from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, FileGroupAlreadyTagged, UsedTag
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile, Tag
@@ -15,8 +15,8 @@ from wrolpi.tags import TagFile, Tag
@pytest.mark.asyncio
async def test_tags_file_group_json(test_async_client, test_session, make_files_structure, tag_factory, example_pdf):
"""The tags of a file are returned in its JSON."""
- tag_one = tag_factory()
- tag_two = tag_factory()
+ tag_one = await tag_factory()
+ tag_two = await tag_factory()
await files_lib.refresh_files()
file_group: FileGroup = test_session.query(FileGroup).one()
@@ -37,7 +37,7 @@ async def test_tags_file_group_json(test_async_client, test_session, make_files_
@pytest.mark.asyncio
async def test_tags_model(test_async_client, test_session, make_files_structure, tag_factory, example_pdf):
"""Can get Tag using class methods."""
- tag1 = tag_factory()
+ tag1 = await tag_factory()
assert tag1.__json__()
assert Tag.find_by_id(tag1.id)
@@ -63,7 +63,7 @@ async def test_tags_file_group(test_async_client, test_session, make_files_struc
await files_lib.refresh_files()
video_group: FileGroup = test_session.query(FileGroup).one()
- tag_one = tag_factory()
+ tag_one = await tag_factory()
assert len(tag_one.tag_files) == 0
Tag.tag_file_group(video_group.id, tag_one.id)
@@ -71,7 +71,7 @@ async def test_tags_file_group(test_async_client, test_session, make_files_struc
assert len(tag_one.tag_files) == 1
assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', ]
- tag_two = tag_factory()
+ tag_two = await tag_factory()
Tag.tag_file_group(video_group.id, tag_two.id)
test_session.commit()
assert len(tag_one.tag_files) == 1
@@ -95,8 +95,8 @@ async def test_tags_config_file(test_async_client, test_session, test_directory,
await files_lib.refresh_files()
pdf: FileGroup = FileGroup.get_by_path(example_pdf, test_session)
video: FileGroup = FileGroup.get_by_path(video_file, test_session)
- tag1 = tag_factory()
- tag2 = tag_factory()
+ tag1 = await tag_factory()
+ tag2 = await tag_factory()
test_session.commit()
tags.save_tags_config(test_session)
@@ -172,11 +172,17 @@ async def test_tags_crud(test_async_client, test_session, example_pdf, assert_ta
dict(name='ガーデン', color='#000000', id=1, file_group_count=1, zim_entry_count=0)]
assert_tags_config(tags={'ガーデン': {'color': '#000000'}})
+ # Tag color can be changed
+ content = dict(name='ガーデン', color='#ffffff')
+ request, response = await test_async_client.post('/api/tag/1', content=json.dumps(content))
+ assert response.status_code == HTTPStatus.OK
+ assert Tag.find_by_name('ガーデン').color == '#ffffff'
+
# Conflicting names return an error.
content = dict(name='ガーデン', color='#111111')
request, response = await test_async_client.post('/api/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
- assert_tags_config(tags={'ガーデン': {'color': '#000000'}})
+ assert_tags_config(tags={'ガーデン': {'color': '#ffffff'}})
# Cannot delete Tag that is used.
request, response = await test_async_client.delete('/api/tag/1')
@@ -197,7 +203,7 @@ async def test_tags_crud(test_async_client, test_session, example_pdf, assert_ta
async def test_delete_tagged_file(test_session, example_pdf, tag_factory):
"""You cannot delete a FileGroup if it is tagged."""
await files_lib.refresh_files()
- tag = tag_factory()
+ tag = await tag_factory()
pdf: FileGroup = test_session.query(FileGroup).one()
pdf.add_tag(tag.id)
@@ -277,7 +283,7 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
from modules.zim import lib as zim_lib
# Create some tags, use tag1.
- tag1, tag2, tag3, tag4 = tag_factory(), tag_factory(), tag_factory(), tag_factory()
+ tag1, tag2, tag3, tag4 = await tag_factory(), await tag_factory(), await tag_factory(), await tag_factory()
# use tag1 for FileGroup.
foo, = make_files_structure({'foo': 'text'})
foo = FileGroup.from_paths(test_session, foo)
@@ -324,16 +330,13 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
@pytest.mark.asyncio
async def test_invalid_tag(test_async_client, test_session, test_directory):
with pytest.raises(InvalidTag):
- tags.upsert_tag('cannot use comma: ,', '#fff')
+ await tags.upsert_tag('cannot use comma: ,', '#fff')
with pytest.raises(InvalidTag):
- tags.upsert_tag('invalid color', 'foo')
+ await tags.upsert_tag('invalid color', 'foo')
with pytest.raises(UnknownTag):
- tags.upsert_tag('Tag ID does not exist', '#fff', 1)
-
- with pytest.raises(UnknownTag):
- tags.delete_tag(1)
+ await tags.upsert_tag('Tag ID does not exist', '#fff', 1)
@pytest.mark.asyncio
@@ -347,7 +350,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
(test_directory / 'tags/cannot be deleted').mkdir(parents=True)
(test_directory / 'tags/cannot be deleted/because of this file').touch()
- tag1, tag2 = tag_factory('First Aid'), tag_factory('Special/name')
+ tag1, tag2 = await tag_factory('First Aid'), await tag_factory('Special/name')
vid1 = video_factory(with_video_file=test_directory / 'vid1.mp4', with_caption_file=True, with_info_json=True,
with_poster_ext='jpg')
vid2 = video_factory(with_video_file=test_directory / 'vid2.mp4', with_info_json=True)
@@ -443,7 +446,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
async def test_update_tag(test_async_client, test_session, test_directory, video_factory, tag_factory):
"""Linked files in the Tag Directory are moved when the tag is renamed."""
# Create tagged Video.
- tag = tag_factory()
+ tag = await tag_factory()
video_factory(title='video', tag_names=[tag.name])
test_session.commit()
@@ -453,10 +456,33 @@ async def test_update_tag(test_async_client, test_session, test_directory, video
assert (test_directory / 'tags/one/video.mp4').is_file()
# Invalid characters are replaced during rename.
- tags.upsert_tag('new/name%', tag.color, tag.id)
+ await tags.upsert_tag('new/name%', tag.color, tag.id)
# Video file was moved.
assert (test_directory / 'tags').is_dir()
assert (test_directory / 'videos/NO CHANNEL/video.mp4').is_file()
assert not (test_directory / 'tags/one/video.mp4').exists()
assert (test_directory / 'tags/new⧸name/video.mp4').is_file()
+
+
+@pytest.mark.asyncio
+async def test_tag_rename_with_channel(test_async_client, test_session, test_directory, video_factory, tag_factory,
+ channel_factory):
+ tag = await tag_factory()
+ channel = channel_factory(name='Channel Name', tag_name=tag.name)
+ video_factory(title='video', channel_id=channel.id)
+ test_session.commit()
+
+ assert (test_directory / 'tags').is_dir()
+ assert (test_directory / 'videos/one/Channel Name').is_dir()
+ assert (test_directory / 'videos/one/Channel Name/video.mp4').is_file()
+
+ await tag.update_tag('New Tag Name', None, session=test_session)
+
+ assert (test_directory / 'tags').is_dir()
+ assert (test_directory / 'videos/New Tag Name/Channel Name').is_dir()
+ assert (test_directory / 'videos/New Tag Name/Channel Name/video.mp4').is_file()
+
+ # Cannot delete Tag when used by Channel. This prevents the need to move Channel directories when Tags are deleted.
+ with pytest.raises(UsedTag):
+ tag.delete()